2024-06-01T02:07:20.063 INFO:root:teuthology version: 0.0.1.dev303+g861a8dc 2024-06-01T02:07:20.063 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi --verbose --archive /home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399 --description orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399/orig.config.yaml 2024-06-01T02:07:20.102 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-06-01T02:07:20.182 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7736399' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 3d89accfe3201286f000e065a1faea2a88c7a6ba ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 3d89accfe3201286f000e065a1faea2a88c7a6ba selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: 6dca6333905c2b38b02396dfd1a76a97d29c2ee7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 3011 sha1: 3d89accfe3201286f000e065a1faea2a88c7a6ba sleep_before_teardown: 0 subset: 21/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_6dca6333905c2b38b02396dfd1a76a97d29c2ee7/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 6dca6333905c2b38b02396dfd1a76a97d29c2ee7 targets: smithi003.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEeVmzFgHtpsazTGOSf3zDNSsnRucbbpHXVXUlC39cub5hEq4ytfyOok2AIyya11wQnDCef954IaikQOwUcTKA8= smithi027.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLenVA0gTi38iawmR1sB157N3F3BgxmhnoixhYW+YgcqiXri/ssp7HUxBzzK4RCsrqPYzLK3eg1xBLMEjoi9akA= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 861a8dcf7aa816a26e13f039336f7ed0a9aec0fa timestamp: 2024-05-31_21:08:02 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2698841 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-06-01T02:07:20.183 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_6dca6333905c2b38b02396dfd1a76a97d29c2ee7/qa; will attempt to use it 2024-06-01T02:07:20.183 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_6dca6333905c2b38b02396dfd1a76a97d29c2ee7/qa/tasks 2024-06-01T02:07:20.184 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-06-01T02:07:20.185 INFO:teuthology.task.internal:Checking packages... 2024-06-01T02:07:20.210 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '3d89accfe3201286f000e065a1faea2a88c7a6ba' 2024-06-01T02:07:20.210 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-06-01T02:07:20.210 INFO:teuthology.packaging:ref: None 2024-06-01T02:07:20.210 INFO:teuthology.packaging:tag: None 2024-06-01T02:07:20.210 INFO:teuthology.packaging:branch: squid 2024-06-01T02:07:20.210 INFO:teuthology.packaging:sha1: 3d89accfe3201286f000e065a1faea2a88c7a6ba 2024-06-01T02:07:20.210 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-06-01T02:07:20.434 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2946.g3d89accf 2024-06-01T02:07:20.436 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-06-01T02:07:20.442 INFO:teuthology.task.internal:no buildpackages task found 2024-06-01T02:07:20.443 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-06-01T02:07:20.449 INFO:teuthology.task.internal:Saving configuration 2024-06-01T02:07:20.469 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-06-01T02:07:20.476 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-06-01T02:07:20.499 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi003.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-06-01 02:01:47.285003', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEeVmzFgHtpsazTGOSf3zDNSsnRucbbpHXVXUlC39cub5hEq4ytfyOok2AIyya11wQnDCef954IaikQOwUcTKA8='} 2024-06-01T02:07:20.524 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi027.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-06-01 02:01:47.283061', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLenVA0gTi38iawmR1sB157N3F3BgxmhnoixhYW+YgcqiXri/ssp7HUxBzzK4RCsrqPYzLK3eg1xBLMEjoi9akA='} 2024-06-01T02:07:20.524 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-06-01T02:07:20.530 INFO:teuthology.task.internal:roles: ubuntu@smithi003.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-06-01T02:07:20.530 INFO:teuthology.task.internal:roles: ubuntu@smithi027.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-06-01T02:07:20.531 INFO:teuthology.run_tasks:Running task console_log... 2024-06-01T02:07:20.590 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f9e22523ac0>, signals=[15]) 2024-06-01T02:07:20.590 INFO:teuthology.run_tasks:Running task internal.connect... 2024-06-01T02:07:20.596 INFO:teuthology.task.internal:Opening connections... 2024-06-01T02:07:20.596 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:07:20.597 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi003.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:07:20.711 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi027.front.sepia.ceph.com 2024-06-01T02:07:20.712 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:07:20.787 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-06-01T02:07:20.794 DEBUG:teuthology.orchestra.run.smithi003:> uname -m 2024-06-01T02:07:20.813 INFO:teuthology.orchestra.run.smithi003.stdout:x86_64 2024-06-01T02:07:20.813 DEBUG:teuthology.orchestra.run.smithi003:> cat /etc/os-release 2024-06-01T02:07:20.871 INFO:teuthology.orchestra.run.smithi003.stdout:NAME="CentOS Stream" 2024-06-01T02:07:20.871 INFO:teuthology.orchestra.run.smithi003.stdout:VERSION="9" 2024-06-01T02:07:20.871 INFO:teuthology.orchestra.run.smithi003.stdout:ID="centos" 2024-06-01T02:07:20.871 INFO:teuthology.orchestra.run.smithi003.stdout:ID_LIKE="rhel fedora" 2024-06-01T02:07:20.871 INFO:teuthology.orchestra.run.smithi003.stdout:VERSION_ID="9" 2024-06-01T02:07:20.871 INFO:teuthology.orchestra.run.smithi003.stdout:PLATFORM_ID="platform:el9" 2024-06-01T02:07:20.871 INFO:teuthology.orchestra.run.smithi003.stdout:PRETTY_NAME="CentOS Stream 9" 2024-06-01T02:07:20.871 INFO:teuthology.orchestra.run.smithi003.stdout:ANSI_COLOR="0;31" 2024-06-01T02:07:20.872 INFO:teuthology.orchestra.run.smithi003.stdout:LOGO="fedora-logo-icon" 2024-06-01T02:07:20.872 INFO:teuthology.orchestra.run.smithi003.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-06-01T02:07:20.872 INFO:teuthology.orchestra.run.smithi003.stdout:HOME_URL="https://centos.org/" 2024-06-01T02:07:20.872 INFO:teuthology.orchestra.run.smithi003.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-06-01T02:07:20.872 INFO:teuthology.orchestra.run.smithi003.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-06-01T02:07:20.872 INFO:teuthology.orchestra.run.smithi003.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-06-01T02:07:20.872 INFO:teuthology.lock.ops:Updating smithi003.front.sepia.ceph.com on lock server 2024-06-01T02:07:20.898 DEBUG:teuthology.orchestra.run.smithi027:> uname -m 2024-06-01T02:07:20.917 INFO:teuthology.orchestra.run.smithi027.stdout:x86_64 2024-06-01T02:07:20.917 DEBUG:teuthology.orchestra.run.smithi027:> cat /etc/os-release 2024-06-01T02:07:20.976 INFO:teuthology.orchestra.run.smithi027.stdout:NAME="CentOS Stream" 2024-06-01T02:07:20.976 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION="9" 2024-06-01T02:07:20.976 INFO:teuthology.orchestra.run.smithi027.stdout:ID="centos" 2024-06-01T02:07:20.976 INFO:teuthology.orchestra.run.smithi027.stdout:ID_LIKE="rhel fedora" 2024-06-01T02:07:20.976 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION_ID="9" 2024-06-01T02:07:20.976 INFO:teuthology.orchestra.run.smithi027.stdout:PLATFORM_ID="platform:el9" 2024-06-01T02:07:20.976 INFO:teuthology.orchestra.run.smithi027.stdout:PRETTY_NAME="CentOS Stream 9" 2024-06-01T02:07:20.976 INFO:teuthology.orchestra.run.smithi027.stdout:ANSI_COLOR="0;31" 2024-06-01T02:07:20.977 INFO:teuthology.orchestra.run.smithi027.stdout:LOGO="fedora-logo-icon" 2024-06-01T02:07:20.977 INFO:teuthology.orchestra.run.smithi027.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-06-01T02:07:20.977 INFO:teuthology.orchestra.run.smithi027.stdout:HOME_URL="https://centos.org/" 2024-06-01T02:07:20.977 INFO:teuthology.orchestra.run.smithi027.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-06-01T02:07:20.977 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-06-01T02:07:20.977 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-06-01T02:07:20.977 INFO:teuthology.lock.ops:Updating smithi027.front.sepia.ceph.com on lock server 2024-06-01T02:07:20.999 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-06-01T02:07:21.007 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-06-01T02:07:21.012 INFO:teuthology.task.internal:Checking for old test directory... 2024-06-01T02:07:21.012 DEBUG:teuthology.orchestra.run.smithi003:> test '!' -e /home/ubuntu/cephtest 2024-06-01T02:07:21.015 DEBUG:teuthology.orchestra.run.smithi027:> test '!' -e /home/ubuntu/cephtest 2024-06-01T02:07:21.033 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-06-01T02:07:21.039 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-06-01T02:07:21.039 DEBUG:teuthology.orchestra.run.smithi003:> test -z $(ls -A /var/lib/ceph) 2024-06-01T02:07:21.072 DEBUG:teuthology.orchestra.run.smithi027:> test -z $(ls -A /var/lib/ceph) 2024-06-01T02:07:21.101 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-06-01T02:07:21.151 INFO:teuthology.run_tasks:Running task kernel... 2024-06-01T02:07:21.165 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-06-01T02:07:21.166 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-06-01T02:07:21.166 DEBUG:teuthology.orchestra.run.smithi003:> test -f /run/.containerenv -o -f /.dockerenv 2024-06-01T02:07:21.166 DEBUG:teuthology.orchestra.run.smithi027:> test -f /run/.containerenv -o -f /.dockerenv 2024-06-01T02:07:21.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-01T02:07:21.183 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2024-06-01T02:07:21.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-01T02:07:21.184 DEBUG:teuthology.orchestra.run.smithi003:> uname -r 2024-06-01T02:07:21.239 INFO:teuthology.orchestra.run.smithi027.stdout:5.14.0-437.el9.x86_64 2024-06-01T02:07:21.240 INFO:teuthology.task.kernel:Running kernel on smithi027: 5.14.0-437.el9.x86_64 2024-06-01T02:07:21.240 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum install -y kernel 2024-06-01T02:07:21.243 INFO:teuthology.orchestra.run.smithi003.stdout:5.14.0-437.el9.x86_64 2024-06-01T02:07:21.243 INFO:teuthology.task.kernel:Running kernel on smithi003: 5.14.0-437.el9.x86_64 2024-06-01T02:07:21.243 DEBUG:teuthology.orchestra.run.smithi003:> sudo yum install -y kernel 2024-06-01T02:07:25.662 INFO:teuthology.orchestra.run.smithi003.stdout:CentOS Stream 9 - BaseOS 2.9 MB/s | 8.1 MB 00:02 2024-06-01T02:07:27.688 INFO:teuthology.orchestra.run.smithi003.stdout:CentOS Stream 9 - AppStream 22 MB/s | 19 MB 00:00 2024-06-01T02:07:27.732 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - BaseOS 1.7 MB/s | 8.1 MB 00:04 2024-06-01T02:07:32.330 INFO:teuthology.orchestra.run.smithi003.stdout:CentOS Stream 9 - CRB 6.5 MB/s | 6.2 MB 00:00 2024-06-01T02:07:33.831 INFO:teuthology.orchestra.run.smithi003.stdout:CentOS Stream 9 - Extras packages 59 kB/s | 17 kB 00:00 2024-06-01T02:07:35.551 INFO:teuthology.orchestra.run.smithi003.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-06-01T02:07:40.149 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - AppStream 1.7 MB/s | 19 MB 00:11 2024-06-01T02:07:40.932 INFO:teuthology.orchestra.run.smithi003.stdout:lab-extras 21 kB/s | 1.7 kB 00:00 2024-06-01T02:07:42.988 INFO:teuthology.orchestra.run.smithi003.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-06-01T02:07:42.988 INFO:teuthology.orchestra.run.smithi003.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-06-01T02:07:43.089 INFO:teuthology.orchestra.run.smithi003.stdout:Dependencies resolved. 2024-06-01T02:07:43.105 INFO:teuthology.orchestra.run.smithi003.stdout:================================================================================ 2024-06-01T02:07:43.105 INFO:teuthology.orchestra.run.smithi003.stdout: Package Architecture Version Repository Size 2024-06-01T02:07:43.105 INFO:teuthology.orchestra.run.smithi003.stdout:================================================================================ 2024-06-01T02:07:43.105 INFO:teuthology.orchestra.run.smithi003.stdout:Installing: 2024-06-01T02:07:43.105 INFO:teuthology.orchestra.run.smithi003.stdout: kernel x86_64 5.14.0-452.el9 baseos 893 k 2024-06-01T02:07:43.105 INFO:teuthology.orchestra.run.smithi003.stdout:Installing dependencies: 2024-06-01T02:07:43.105 INFO:teuthology.orchestra.run.smithi003.stdout: kernel-core x86_64 5.14.0-452.el9 baseos 16 M 2024-06-01T02:07:43.106 INFO:teuthology.orchestra.run.smithi003.stdout: kernel-modules x86_64 5.14.0-452.el9 baseos 35 M 2024-06-01T02:07:43.106 INFO:teuthology.orchestra.run.smithi003.stdout: kernel-modules-core x86_64 5.14.0-452.el9 baseos 29 M 2024-06-01T02:07:43.106 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:07:43.106 INFO:teuthology.orchestra.run.smithi003.stdout:Transaction Summary 2024-06-01T02:07:43.106 INFO:teuthology.orchestra.run.smithi003.stdout:================================================================================ 2024-06-01T02:07:43.106 INFO:teuthology.orchestra.run.smithi003.stdout:Install 4 Packages 2024-06-01T02:07:43.106 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:07:43.107 INFO:teuthology.orchestra.run.smithi003.stdout:Total download size: 80 M 2024-06-01T02:07:43.107 INFO:teuthology.orchestra.run.smithi003.stdout:Installed size: 124 M 2024-06-01T02:07:43.107 INFO:teuthology.orchestra.run.smithi003.stdout:Downloading Packages: 2024-06-01T02:07:44.583 INFO:teuthology.orchestra.run.smithi003.stdout:(1/4): kernel-5.14.0-452.el9.x86_64.rpm 632 kB/s | 893 kB 00:01 2024-06-01T02:07:45.375 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - CRB 4.1 MB/s | 6.2 MB 00:01 2024-06-01T02:07:46.700 INFO:teuthology.orchestra.run.smithi003.stdout:(2/4): kernel-modules-5.14.0-452.el9.x86_64.rpm 9.8 MB/s | 35 MB 00:03 2024-06-01T02:07:46.975 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - Extras packages 44 kB/s | 17 kB 00:00 2024-06-01T02:07:48.642 INFO:teuthology.orchestra.run.smithi003.stdout:(3/4): kernel-core-5.14.0-452.el9.x86_64.rpm 2.9 MB/s | 16 MB 00:05 2024-06-01T02:07:49.238 INFO:teuthology.orchestra.run.smithi027.stdout:Extra Packages for Enterprise Linux 10 MB/s | 21 MB 00:02 2024-06-01T02:07:50.883 INFO:teuthology.orchestra.run.smithi003.stdout:(4/4): kernel-modules-core-5.14.0-452.el9.x86_6 4.6 MB/s | 29 MB 00:06 2024-06-01T02:07:50.883 INFO:teuthology.orchestra.run.smithi003.stdout:-------------------------------------------------------------------------------- 2024-06-01T02:07:50.883 INFO:teuthology.orchestra.run.smithi003.stdout:Total 10 MB/s | 80 MB 00:07 2024-06-01T02:07:51.608 INFO:teuthology.orchestra.run.smithi003.stdout:Running transaction check 2024-06-01T02:07:52.050 INFO:teuthology.orchestra.run.smithi003.stdout:Transaction check succeeded. 2024-06-01T02:07:52.050 INFO:teuthology.orchestra.run.smithi003.stdout:Running transaction test 2024-06-01T02:07:53.025 INFO:teuthology.orchestra.run.smithi003.stdout:Transaction test succeeded. 2024-06-01T02:07:53.025 INFO:teuthology.orchestra.run.smithi003.stdout:Running transaction 2024-06-01T02:07:54.568 INFO:teuthology.orchestra.run.smithi027.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-06-01T02:07:55.563 INFO:teuthology.orchestra.run.smithi003.stdout: Preparing : 1/1 2024-06-01T02:07:56.220 INFO:teuthology.orchestra.run.smithi003.stdout: Installing : kernel-modules-core-5.14.0-452.el9.x86_64 1/4 2024-06-01T02:07:56.308 INFO:teuthology.orchestra.run.smithi003.stdout: Installing : kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-01T02:07:56.641 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-06-01T02:07:56.641 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-06-01T02:07:56.744 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-06-01T02:07:56.760 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: kernel x86_64 5.14.0-452.el9 baseos 893 k 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-core x86_64 5.14.0-452.el9 baseos 16 M 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules x86_64 5.14.0-452.el9 baseos 35 M 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-core x86_64 5.14.0-452.el9 baseos 29 M 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-06-01T02:07:56.761 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-06-01T02:07:56.762 INFO:teuthology.orchestra.run.smithi027.stdout:Install 4 Packages 2024-06-01T02:07:56.762 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:07:56.762 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 80 M 2024-06-01T02:07:56.762 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 124 M 2024-06-01T02:07:56.762 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-06-01T02:07:57.570 INFO:teuthology.orchestra.run.smithi027.stdout:(1/4): kernel-5.14.0-452.el9.x86_64.rpm 1.1 MB/s | 893 kB 00:00 2024-06-01T02:07:59.105 INFO:teuthology.orchestra.run.smithi003.stdout: Running scriptlet: kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-01T02:07:59.261 INFO:teuthology.orchestra.run.smithi003.stdout: Installing : kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-01T02:08:04.362 INFO:teuthology.orchestra.run.smithi003.stdout: Running scriptlet: kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-01T02:08:04.409 INFO:teuthology.orchestra.run.smithi003.stdout: Installing : kernel-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:08:09.519 INFO:teuthology.orchestra.run.smithi003.stdout: Running scriptlet: kernel-modules-core-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:08:12.602 INFO:teuthology.orchestra.run.smithi027.stdout:(2/4): kernel-core-5.14.0-452.el9.x86_64.rpm 1.0 MB/s | 16 MB 00:15 2024-06-01T02:08:20.493 INFO:teuthology.orchestra.run.smithi027.stdout:(3/4): kernel-modules-5.14.0-452.el9.x86_64.rpm 1.5 MB/s | 35 MB 00:23 2024-06-01T02:08:30.594 INFO:teuthology.orchestra.run.smithi027.stdout:(4/4): kernel-modules-core-5.14.0-452.el9.x86_6 895 kB/s | 29 MB 00:33 2024-06-01T02:08:30.594 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-06-01T02:08:30.594 INFO:teuthology.orchestra.run.smithi027.stdout:Total 2.4 MB/s | 80 MB 00:33 2024-06-01T02:08:31.316 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-06-01T02:08:31.752 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-06-01T02:08:31.752 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-06-01T02:08:32.710 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-06-01T02:08:32.710 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-06-01T02:08:35.266 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-06-01T02:08:35.923 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-modules-core-5.14.0-452.el9.x86_64 1/4 2024-06-01T02:08:36.018 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-01T02:08:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-01T02:08:38.956 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-01T02:08:40.533 INFO:teuthology.orchestra.run.smithi003.stdout: Running scriptlet: kernel-core-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:08:40.576 INFO:teuthology.orchestra.run.smithi003.stdout: Running scriptlet: kernel-modules-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:08:40.932 INFO:teuthology.orchestra.run.smithi003.stdout: Running scriptlet: kernel-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:08:40.932 INFO:teuthology.orchestra.run.smithi003.stdout: Verifying : kernel-5.14.0-452.el9.x86_64 1/4 2024-06-01T02:08:40.932 INFO:teuthology.orchestra.run.smithi003.stdout: Verifying : kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-01T02:08:40.932 INFO:teuthology.orchestra.run.smithi003.stdout: Verifying : kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-01T02:08:41.745 INFO:teuthology.orchestra.run.smithi003.stdout: Verifying : kernel-modules-core-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:08:41.745 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:08:41.745 INFO:teuthology.orchestra.run.smithi003.stdout:Installed: 2024-06-01T02:08:41.745 INFO:teuthology.orchestra.run.smithi003.stdout: kernel-5.14.0-452.el9.x86_64 2024-06-01T02:08:41.745 INFO:teuthology.orchestra.run.smithi003.stdout: kernel-core-5.14.0-452.el9.x86_64 2024-06-01T02:08:41.745 INFO:teuthology.orchestra.run.smithi003.stdout: kernel-modules-5.14.0-452.el9.x86_64 2024-06-01T02:08:41.745 INFO:teuthology.orchestra.run.smithi003.stdout: kernel-modules-core-5.14.0-452.el9.x86_64 2024-06-01T02:08:41.746 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:08:41.746 INFO:teuthology.orchestra.run.smithi003.stdout:Complete! 2024-06-01T02:08:42.104 DEBUG:teuthology.orchestra.run.smithi003:> rpm -q kernel | sort -rV | head -n 1 2024-06-01T02:08:42.216 INFO:teuthology.orchestra.run.smithi003.stdout:kernel-5.14.0-452.el9.x86_64 2024-06-01T02:08:42.217 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-452.el9.x86_64 2024-06-01T02:08:42.217 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-452.el9.x86_64 2024-06-01T02:08:42.217 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-06-01T02:08:42.217 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-06-01T02:08:42.218 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-06-01T02:08:42.218 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi003.front.sepia.ceph.com, path=None, version=distro) 2024-06-01T02:08:42.218 DEBUG:teuthology.orchestra.run.smithi003:> rpm -q kernel | sort -rV | head -n 1 2024-06-01T02:08:42.271 INFO:teuthology.orchestra.run.smithi003.stdout:kernel-5.14.0-452.el9.x86_64 2024-06-01T02:08:42.272 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-452.el9.x86_64 2024-06-01T02:08:42.272 DEBUG:teuthology.orchestra.run.smithi003:> sudo rpm -qi grub2-tools 2024-06-01T02:08:42.313 INFO:teuthology.orchestra.run.smithi003.stdout:Name : grub2-tools 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Epoch : 1 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Version : 2.06 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Release : 68.el9 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Architecture: x86_64 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Group : Unspecified 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Size : 8264254 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:License : GPLv3+ 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-06-01T02:08:42.314 INFO:teuthology.orchestra.run.smithi003.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:Packager : builder@centos.org 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:Vendor : CentOS 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:URL : http://www.gnu.org/software/grub/ 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:Summary : Support tools for GRUB. 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:Description : 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:customizable bootloader with modular architecture. It supports a rich 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:variety of kernel formats, file systems, computer architectures and 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:hardware devices. 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:08:42.315 INFO:teuthology.orchestra.run.smithi003.stdout:This subpackage provides tools for support of all platforms. 2024-06-01T02:08:42.316 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-06-01T02:08:42.316 INFO:teuthology.task.kernel:Updating grub on smithi003 to boot 5.14.0-452.el9.x86_64 2024-06-01T02:08:42.317 DEBUG:teuthology.orchestra.run.smithi003:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-06-01T02:08:42.947 INFO:teuthology.orchestra.run.smithi003.stderr:Generating grub configuration file ... 2024-06-01T02:08:44.046 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-01T02:08:44.095 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:08:44.433 INFO:teuthology.orchestra.run.smithi003.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-06-01T02:08:44.450 INFO:teuthology.orchestra.run.smithi003.stderr:done 2024-06-01T02:08:44.453 DEBUG:teuthology.orchestra.run.smithi003:> mktemp 2024-06-01T02:08:44.473 INFO:teuthology.orchestra.run.smithi003.stdout:/tmp/tmp.ZPTEkYJx3B 2024-06-01T02:08:44.474 DEBUG:teuthology.orchestra.run.smithi003:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ZPTEkYJx3B 2024-06-01T02:08:44.558 DEBUG:teuthology.orchestra.run.smithi003:> sudo chmod 0666 /tmp/tmp.ZPTEkYJx3B 2024-06-01T02:08:44.669 DEBUG:teuthology.orchestra.remote:smithi003:/tmp/tmp.ZPTEkYJx3B is 6KB 2024-06-01T02:08:44.718 DEBUG:teuthology.orchestra.run.smithi003:> rm -fr /tmp/tmp.ZPTEkYJx3B 2024-06-01T02:08:44.733 DEBUG:teuthology.orchestra.run.smithi003:> sudo /bin/ls /boot/loader/entries || true 2024-06-01T02:08:44.800 INFO:teuthology.orchestra.run.smithi003.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-06-01T02:08:44.800 INFO:teuthology.orchestra.run.smithi003.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-06-01T02:08:44.800 INFO:teuthology.orchestra.run.smithi003.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-06-01T02:08:44.800 INFO:teuthology.orchestra.run.smithi003.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-452.el9.x86_64.conf 2024-06-01T02:08:44.802 DEBUG:teuthology.orchestra.run.smithi003:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-452.el9.x86_64 2024-06-01T02:08:44.942 DEBUG:teuthology.orchestra.run.smithi003:> sudo shutdown -r now 2024-06-01T02:08:49.208 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-core-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:09:14.945 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-06-01T02:09:14.949 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi003.front.sepia.ceph.com' 2024-06-01T02:09:14.950 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi003.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:09:19.934 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-core-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:09:19.975 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:09:20.317 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:09:20.317 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-5.14.0-452.el9.x86_64 1/4 2024-06-01T02:09:20.317 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-01T02:09:20.317 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-01T02:09:21.114 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-modules-core-5.14.0-452.el9.x86_64 4/4 2024-06-01T02:09:21.114 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:09:21.114 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-06-01T02:09:21.114 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-5.14.0-452.el9.x86_64 2024-06-01T02:09:21.114 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-core-5.14.0-452.el9.x86_64 2024-06-01T02:09:21.114 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-5.14.0-452.el9.x86_64 2024-06-01T02:09:21.115 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-core-5.14.0-452.el9.x86_64 2024-06-01T02:09:21.115 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:09:21.115 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-06-01T02:09:21.478 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2024-06-01T02:09:21.560 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-5.14.0-452.el9.x86_64 2024-06-01T02:09:21.561 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-452.el9.x86_64 2024-06-01T02:09:21.561 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-452.el9.x86_64 2024-06-01T02:09:21.561 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-06-01T02:09:21.561 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-06-01T02:09:21.561 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-06-01T02:09:21.561 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi027.front.sepia.ceph.com, path=None, version=distro) 2024-06-01T02:09:21.561 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2024-06-01T02:09:21.615 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-5.14.0-452.el9.x86_64 2024-06-01T02:09:21.616 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-452.el9.x86_64 2024-06-01T02:09:21.616 DEBUG:teuthology.orchestra.run.smithi027:> sudo rpm -qi grub2-tools 2024-06-01T02:09:21.647 INFO:teuthology.orchestra.run.smithi027.stdout:Name : grub2-tools 2024-06-01T02:09:21.647 INFO:teuthology.orchestra.run.smithi027.stdout:Epoch : 1 2024-06-01T02:09:21.647 INFO:teuthology.orchestra.run.smithi027.stdout:Version : 2.06 2024-06-01T02:09:21.647 INFO:teuthology.orchestra.run.smithi027.stdout:Release : 68.el9 2024-06-01T02:09:21.647 INFO:teuthology.orchestra.run.smithi027.stdout:Architecture: x86_64 2024-06-01T02:09:21.647 INFO:teuthology.orchestra.run.smithi027.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Group : Unspecified 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Size : 8264254 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:License : GPLv3+ 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Packager : builder@centos.org 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Vendor : CentOS 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:URL : http://www.gnu.org/software/grub/ 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Summary : Support tools for GRUB. 2024-06-01T02:09:21.648 INFO:teuthology.orchestra.run.smithi027.stdout:Description : 2024-06-01T02:09:21.649 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:09:21.649 INFO:teuthology.orchestra.run.smithi027.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-06-01T02:09:21.649 INFO:teuthology.orchestra.run.smithi027.stdout:customizable bootloader with modular architecture. It supports a rich 2024-06-01T02:09:21.649 INFO:teuthology.orchestra.run.smithi027.stdout:variety of kernel formats, file systems, computer architectures and 2024-06-01T02:09:21.649 INFO:teuthology.orchestra.run.smithi027.stdout:hardware devices. 2024-06-01T02:09:21.649 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:09:21.649 INFO:teuthology.orchestra.run.smithi027.stdout:This subpackage provides tools for support of all platforms. 2024-06-01T02:09:21.650 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-06-01T02:09:21.650 INFO:teuthology.task.kernel:Updating grub on smithi027 to boot 5.14.0-452.el9.x86_64 2024-06-01T02:09:21.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-06-01T02:09:22.257 INFO:teuthology.orchestra.run.smithi027.stderr:Generating grub configuration file ... 2024-06-01T02:09:23.748 INFO:teuthology.orchestra.run.smithi027.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-06-01T02:09:23.766 INFO:teuthology.orchestra.run.smithi027.stderr:done 2024-06-01T02:09:23.768 DEBUG:teuthology.orchestra.run.smithi027:> mktemp 2024-06-01T02:09:23.784 INFO:teuthology.orchestra.run.smithi027.stdout:/tmp/tmp.b02VGMq8PS 2024-06-01T02:09:23.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.b02VGMq8PS 2024-06-01T02:09:23.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 0666 /tmp/tmp.b02VGMq8PS 2024-06-01T02:09:23.991 DEBUG:teuthology.orchestra.remote:smithi027:/tmp/tmp.b02VGMq8PS is 6KB 2024-06-01T02:09:24.040 DEBUG:teuthology.orchestra.run.smithi027:> rm -fr /tmp/tmp.b02VGMq8PS 2024-06-01T02:09:24.055 DEBUG:teuthology.orchestra.run.smithi027:> sudo /bin/ls /boot/loader/entries || true 2024-06-01T02:09:24.117 INFO:teuthology.orchestra.run.smithi027.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-06-01T02:09:24.117 INFO:teuthology.orchestra.run.smithi027.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-06-01T02:09:24.117 INFO:teuthology.orchestra.run.smithi027.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-06-01T02:09:24.117 INFO:teuthology.orchestra.run.smithi027.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-452.el9.x86_64.conf 2024-06-01T02:09:24.118 DEBUG:teuthology.orchestra.run.smithi027:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-452.el9.x86_64 2024-06-01T02:09:24.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo shutdown -r now 2024-06-01T02:09:54.294 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-06-01T02:09:54.295 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-01T02:09:54.296 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:10:14.971 DEBUG:teuthology.orchestra.remote:timed out 2024-06-01T02:10:23.980 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi003.front.sepia.ceph.com' 2024-06-01T02:10:23.981 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi003.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:10:24.471 DEBUG:teuthology.orchestra.run.smithi003:> true 2024-06-01T02:10:24.981 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi003.front.sepia.ceph.com' 2024-06-01T02:10:24.981 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-452.el9.x86_64"... 2024-06-01T02:10:24.982 DEBUG:teuthology.orchestra.run.smithi003:> uname -r 2024-06-01T02:10:24.997 INFO:teuthology.orchestra.run.smithi003.stdout:5.14.0-452.el9.x86_64 2024-06-01T02:10:24.997 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-452.el9.x86_64 vs 5.14.0-452.el9.x86_64 2024-06-01T02:10:24.997 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-06-01T02:10:24.997 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-06-01T02:10:25.998 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-06-01T02:10:25.999 DEBUG:teuthology.orchestra.run.smithi003:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-06-01T02:10:26.080 INFO:teuthology.orchestra.run.smithi003.stdout:ttyS1 2024-06-01T02:10:26.125 DEBUG:teuthology.parallel:result is None 2024-06-01T02:10:28.766 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-06-01T02:10:37.767 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-01T02:10:37.768 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:10:40.830 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-06-01T02:10:52.831 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-01T02:10:52.832 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:11:11.262 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-06-01T02:11:26.263 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-01T02:11:26.264 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:11:26.625 DEBUG:teuthology.orchestra.run.smithi027:> true 2024-06-01T02:11:27.062 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-01T02:11:27.062 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-452.el9.x86_64"... 2024-06-01T02:11:27.063 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2024-06-01T02:11:27.078 INFO:teuthology.orchestra.run.smithi027.stdout:5.14.0-452.el9.x86_64 2024-06-01T02:11:27.078 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-452.el9.x86_64 vs 5.14.0-452.el9.x86_64 2024-06-01T02:11:27.078 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-06-01T02:11:27.078 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-06-01T02:11:28.079 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-06-01T02:11:28.079 DEBUG:teuthology.orchestra.run.smithi027:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-06-01T02:11:28.150 INFO:teuthology.orchestra.run.smithi027.stdout:ttyS1 2024-06-01T02:11:28.189 DEBUG:teuthology.parallel:result is None 2024-06-01T02:11:28.189 INFO:teuthology.run_tasks:Running task internal.base... 2024-06-01T02:11:28.196 INFO:teuthology.task.internal:Creating test directory... 2024-06-01T02:11:28.196 DEBUG:teuthology.orchestra.run.smithi003:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-06-01T02:11:28.200 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-06-01T02:11:28.217 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-06-01T02:11:28.237 INFO:teuthology.run_tasks:Running task internal.archive... 2024-06-01T02:11:28.252 INFO:teuthology.task.internal:Creating archive directory... 2024-06-01T02:11:28.252 DEBUG:teuthology.orchestra.run.smithi003:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-06-01T02:11:28.261 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-06-01T02:11:28.303 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-06-01T02:11:28.310 INFO:teuthology.task.internal:Enabling coredump saving... 2024-06-01T02:11:28.310 DEBUG:teuthology.orchestra.run.smithi003:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-06-01T02:11:28.342 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-06-01T02:11:28.381 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-01T02:11:28.382 INFO:teuthology.orchestra.run.smithi003.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-01T02:11:28.391 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-01T02:11:28.395 INFO:teuthology.orchestra.run.smithi003.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-01T02:11:28.397 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-06-01T02:11:28.404 INFO:teuthology.task.internal:Configuring sudo... 2024-06-01T02:11:28.404 DEBUG:teuthology.orchestra.run.smithi003:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-06-01T02:11:28.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-06-01T02:11:28.467 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-06-01T02:11:28.476 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-06-01T02:11:28.476 DEBUG:teuthology.orchestra.run.smithi003:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-06-01T02:11:28.510 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-06-01T02:11:28.526 DEBUG:teuthology.orchestra.run.smithi003:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-01T02:11:28.609 DEBUG:teuthology.orchestra.run.smithi003:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-01T02:11:28.662 DEBUG:teuthology.orchestra.run.smithi003:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-01T02:11:28.746 DEBUG:teuthology.orchestra.run.smithi003:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-01T02:11:28.789 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:11:28.789 DEBUG:teuthology.orchestra.run.smithi003:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-06-01T02:11:28.856 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-01T02:11:28.898 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-01T02:11:28.947 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-01T02:11:29.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-01T02:11:29.070 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-01T02:11:29.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-06-01T02:11:29.135 DEBUG:teuthology.orchestra.run.smithi003:> sudo service rsyslog restart 2024-06-01T02:11:29.137 DEBUG:teuthology.orchestra.run.smithi027:> sudo service rsyslog restart 2024-06-01T02:11:29.183 INFO:teuthology.orchestra.run.smithi003.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-01T02:11:29.228 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-01T02:11:29.624 INFO:teuthology.run_tasks:Running task internal.timer... 2024-06-01T02:11:29.632 INFO:teuthology.task.internal:Starting timer... 2024-06-01T02:11:29.632 INFO:teuthology.run_tasks:Running task pcp... 2024-06-01T02:11:29.643 INFO:teuthology.run_tasks:Running task selinux... 2024-06-01T02:11:29.651 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-06-01T02:11:29.651 DEBUG:teuthology.orchestra.run.smithi003:> sudo service auditd rotate 2024-06-01T02:11:29.741 INFO:teuthology.orchestra.run.smithi003.stdout:Rotating logs: 2024-06-01T02:11:29.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo service auditd rotate 2024-06-01T02:11:29.834 INFO:teuthology.orchestra.run.smithi027.stdout:Rotating logs: 2024-06-01T02:11:29.838 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-06-01T02:11:29.839 DEBUG:teuthology.orchestra.run.smithi003:> /usr/sbin/getenforce 2024-06-01T02:11:29.886 INFO:teuthology.orchestra.run.smithi003.stdout:Permissive 2024-06-01T02:11:29.886 DEBUG:teuthology.orchestra.run.smithi027:> /usr/sbin/getenforce 2024-06-01T02:11:29.940 INFO:teuthology.orchestra.run.smithi027.stdout:Permissive 2024-06-01T02:11:29.940 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi003.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi027.front.sepia.ceph.com': 'permissive'} 2024-06-01T02:11:29.940 DEBUG:teuthology.orchestra.run.smithi003:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-06-01T02:11:29.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-01T02:11:29.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-06-01T02:11:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-01T02:11:30.042 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-06-01T02:11:30.042 DEBUG:teuthology.orchestra.run.smithi003:> sudo /usr/sbin/setenforce permissive 2024-06-01T02:11:30.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo /usr/sbin/setenforce permissive 2024-06-01T02:11:30.366 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-06-01T02:11:30.377 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-06-01T02:11:30.388 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-06-01T02:11:30.389 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi003.front.sepia.ceph.com,smithi027.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-06-01T02:16:00.753 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi003.front.sepia.ceph.com'), Remote(name='ubuntu@smithi027.front.sepia.ceph.com')] 2024-06-01T02:16:00.754 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi003.front.sepia.ceph.com' 2024-06-01T02:16:00.755 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi003.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:16:00.831 DEBUG:teuthology.orchestra.run.smithi003:> true 2024-06-01T02:16:00.914 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi003.front.sepia.ceph.com' 2024-06-01T02:16:00.914 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-01T02:16:00.915 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-01T02:16:00.989 DEBUG:teuthology.orchestra.run.smithi027:> true 2024-06-01T02:16:01.068 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-06-01T02:16:01.069 INFO:teuthology.run_tasks:Running task clock... 2024-06-01T02:16:01.082 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-06-01T02:16:01.082 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-06-01T02:16:01.083 DEBUG:teuthology.orchestra.run.smithi003:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-01T02:16:01.085 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-06-01T02:16:01.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-01T02:16:01.114 INFO:teuthology.orchestra.run.smithi003.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-06-01T02:16:01.129 INFO:teuthology.orchestra.run.smithi003.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-06-01T02:16:01.142 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-06-01T02:16:01.155 INFO:teuthology.orchestra.run.smithi003.stderr:sudo: ntpd: command not found 2024-06-01T02:16:01.158 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-06-01T02:16:01.166 INFO:teuthology.orchestra.run.smithi003.stdout:506 Cannot talk to daemon 2024-06-01T02:16:01.181 INFO:teuthology.orchestra.run.smithi003.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-06-01T02:16:01.185 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: ntpd: command not found 2024-06-01T02:16:01.196 INFO:teuthology.orchestra.run.smithi003.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-06-01T02:16:01.197 INFO:teuthology.orchestra.run.smithi027.stdout:506 Cannot talk to daemon 2024-06-01T02:16:01.212 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-06-01T02:16:01.227 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-06-01T02:16:01.255 INFO:teuthology.orchestra.run.smithi003.stderr:bash: line 1: ntpq: command not found 2024-06-01T02:16:01.258 INFO:teuthology.orchestra.run.smithi003.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-01T02:16:01.259 INFO:teuthology.orchestra.run.smithi003.stdout:=============================================================================== 2024-06-01T02:16:01.259 INFO:teuthology.orchestra.run.smithi003.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:16:01.259 INFO:teuthology.orchestra.run.smithi003.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:16:01.259 INFO:teuthology.orchestra.run.smithi003.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:16:01.259 INFO:teuthology.orchestra.run.smithi003.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:16:01.286 INFO:teuthology.orchestra.run.smithi027.stderr:bash: line 1: ntpq: command not found 2024-06-01T02:16:01.289 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-01T02:16:01.289 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2024-06-01T02:16:01.290 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:16:01.290 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:16:01.290 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:16:01.290 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:16:01.290 INFO:teuthology.run_tasks:Running task cephadm... 2024-06-01T02:16:01.379 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '3d89accfe3201286f000e065a1faea2a88c7a6ba'} 2024-06-01T02:16:01.379 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-06-01T02:16:01.380 INFO:tasks.cephadm:Cluster fsid is e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:16:01.380 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-06-01T02:16:01.380 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-06-01T02:16:01.380 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi003': '172.21.15.3', 'mon.smithi027': '172.21.15.27'} 2024-06-01T02:16:01.380 INFO:tasks.cephadm:Normalizing hostnames... 2024-06-01T02:16:01.380 DEBUG:teuthology.orchestra.run.smithi003:> sudo hostname $(hostname -s) 2024-06-01T02:16:01.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo hostname $(hostname -s) 2024-06-01T02:16:01.434 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-06-01T02:16:01.435 DEBUG:teuthology.orchestra.run.smithi003:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-06-01T02:16:01.626 INFO:teuthology.orchestra.run.smithi003.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Jun 1 02:16 /home/ubuntu/cephtest/cephadm 2024-06-01T02:16:01.626 DEBUG:teuthology.orchestra.run.smithi027:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-06-01T02:16:01.817 INFO:teuthology.orchestra.run.smithi027.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Jun 1 02:16 /home/ubuntu/cephtest/cephadm 2024-06-01T02:16:01.817 DEBUG:teuthology.orchestra.run.smithi003:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-06-01T02:16:01.834 DEBUG:teuthology.orchestra.run.smithi027:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-06-01T02:16:01.859 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-06-01T02:16:01.860 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-06-01T02:16:01.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-06-01T02:16:02.794 INFO:teuthology.orchestra.run.smithi003.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-01T02:16:02.846 INFO:teuthology.orchestra.run.smithi027.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-01T02:16:18.927 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-06-01T02:16:18.928 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-06-01T02:16:18.928 INFO:teuthology.orchestra.run.smithi027.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-06-01T02:16:18.928 INFO:teuthology.orchestra.run.smithi027.stdout: "repo_digests": [ 2024-06-01T02:16:18.928 INFO:teuthology.orchestra.run.smithi027.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-06-01T02:16:18.928 INFO:teuthology.orchestra.run.smithi027.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-06-01T02:16:18.928 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-06-01T02:16:18.928 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-06-01T02:16:19.110 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:16:19.110 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-06-01T02:16:19.110 INFO:teuthology.orchestra.run.smithi003.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-06-01T02:16:19.110 INFO:teuthology.orchestra.run.smithi003.stdout: "repo_digests": [ 2024-06-01T02:16:19.110 INFO:teuthology.orchestra.run.smithi003.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-06-01T02:16:19.110 INFO:teuthology.orchestra.run.smithi003.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-06-01T02:16:19.110 INFO:teuthology.orchestra.run.smithi003.stdout: ] 2024-06-01T02:16:19.110 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:16:19.134 DEBUG:teuthology.orchestra.run.smithi003:> sudo mkdir -p /etc/ceph 2024-06-01T02:16:19.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo mkdir -p /etc/ceph 2024-06-01T02:16:19.205 DEBUG:teuthology.orchestra.run.smithi003:> sudo chmod 777 /etc/ceph 2024-06-01T02:16:19.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 777 /etc/ceph 2024-06-01T02:16:19.272 INFO:tasks.cephadm:Writing seed config... 2024-06-01T02:16:19.273 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-06-01T02:16:19.273 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-06-01T02:16:19.273 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-06-01T02:16:19.273 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-06-01T02:16:19.273 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-06-01T02:16:19.273 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-06-01T02:16:19.273 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-06-01T02:16:19.273 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-06-01T02:16:19.274 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:16:19.274 DEBUG:teuthology.orchestra.run.smithi003:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-06-01T02:16:19.302 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = e42641dc-1fbc-11ef-bc9b-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-06-01T02:16:19.302 DEBUG:teuthology.orchestra.run.smithi003:mon.smithi003> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi003.service 2024-06-01T02:16:19.344 INFO:tasks.cephadm:Bootstrapping... 2024-06-01T02:16:19.345 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.3 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-06-01T02:16:19.548 INFO:teuthology.orchestra.run.smithi003.stderr:-------------------------------------------------------------------------------- 2024-06-01T02:16:19.548 INFO:teuthology.orchestra.run.smithi003.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'e42641dc-1fbc-11ef-bc9b-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.3', '--skip-admin-label'] 2024-06-01T02:16:19.572 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: 5.0.2 2024-06-01T02:16:19.574 INFO:teuthology.orchestra.run.smithi003.stderr:Verifying podman|docker is present... 2024-06-01T02:16:19.596 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: 5.0.2 2024-06-01T02:16:19.598 INFO:teuthology.orchestra.run.smithi003.stderr:Verifying lvm2 is present... 2024-06-01T02:16:19.598 INFO:teuthology.orchestra.run.smithi003.stderr:Verifying time synchronization is in place... 2024-06-01T02:16:19.605 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-06-01T02:16:19.612 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: inactive 2024-06-01T02:16:19.619 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: enabled 2024-06-01T02:16:19.625 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: active 2024-06-01T02:16:19.626 INFO:teuthology.orchestra.run.smithi003.stderr:Unit chronyd.service is enabled and running 2024-06-01T02:16:19.626 INFO:teuthology.orchestra.run.smithi003.stderr:Repeating the final host check... 2024-06-01T02:16:19.648 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: 5.0.2 2024-06-01T02:16:19.651 INFO:teuthology.orchestra.run.smithi003.stderr:podman (/bin/podman) version 5.0.2 is present 2024-06-01T02:16:19.651 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl is present 2024-06-01T02:16:19.651 INFO:teuthology.orchestra.run.smithi003.stderr:lvcreate is present 2024-06-01T02:16:19.657 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-06-01T02:16:19.664 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: inactive 2024-06-01T02:16:19.671 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: enabled 2024-06-01T02:16:19.678 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: active 2024-06-01T02:16:19.679 INFO:teuthology.orchestra.run.smithi003.stderr:Unit chronyd.service is enabled and running 2024-06-01T02:16:19.679 INFO:teuthology.orchestra.run.smithi003.stderr:Host looks OK 2024-06-01T02:16:19.679 INFO:teuthology.orchestra.run.smithi003.stderr:Cluster fsid: e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:16:19.679 INFO:teuthology.orchestra.run.smithi003.stderr:Acquiring lock 139744780250992 on /run/cephadm/e42641dc-1fbc-11ef-bc9b-c7b262605968.lock 2024-06-01T02:16:19.680 INFO:teuthology.orchestra.run.smithi003.stderr:Lock 139744780250992 acquired on /run/cephadm/e42641dc-1fbc-11ef-bc9b-c7b262605968.lock 2024-06-01T02:16:19.680 INFO:teuthology.orchestra.run.smithi003.stderr:Verifying IP 172.21.15.3 port 3300 ... 2024-06-01T02:16:19.680 INFO:teuthology.orchestra.run.smithi003.stderr:Verifying IP 172.21.15.3 port 6789 ... 2024-06-01T02:16:19.680 INFO:teuthology.orchestra.run.smithi003.stderr:Base mon IP is 172.21.15.3, final addrv is [v2:172.21.15.3:3300,v1:172.21.15.3:6789] 2024-06-01T02:16:19.683 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.3 metric 100 2024-06-01T02:16:19.683 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.3 metric 100 2024-06-01T02:16:19.685 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-06-01T02:16:19.685 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-06-01T02:16:19.686 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-06-01T02:16:19.688 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-06-01T02:16:19.688 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-06-01T02:16:19.688 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-06-01T02:16:19.688 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-06-01T02:16:19.688 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:15dc/64 scope link noprefixroute 2024-06-01T02:16:19.689 INFO:teuthology.orchestra.run.smithi003.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-06-01T02:16:19.689 INFO:teuthology.orchestra.run.smithi003.stderr:Mon IP `172.21.15.3` is in CIDR network `172.21.0.0/20` 2024-06-01T02:16:19.689 INFO:teuthology.orchestra.run.smithi003.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-06-01T02:16:19.690 INFO:teuthology.orchestra.run.smithi003.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-01T02:16:19.796 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-06-01T02:16:20.207 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: Getting image source signatures 2024-06-01T02:16:20.208 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-06-01T02:16:20.208 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-06-01T02:16:20.209 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-06-01T02:16:20.209 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-06-01T02:16:20.209 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-06-01T02:16:20.209 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-06-01T02:16:20.336 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: Writing manifest to image destination 2024-06-01T02:16:20.456 INFO:teuthology.orchestra.run.smithi003.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-06-01T02:16:20.869 INFO:teuthology.orchestra.run.smithi003.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-06-01T02:16:21.473 INFO:teuthology.orchestra.run.smithi003.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-06-01T02:16:21.473 INFO:teuthology.orchestra.run.smithi003.stderr:Extracting ceph user uid/gid from container image... 2024-06-01T02:16:21.888 INFO:teuthology.orchestra.run.smithi003.stderr:stat: 167 167 2024-06-01T02:16:22.478 INFO:teuthology.orchestra.run.smithi003.stderr:Creating initial keys... 2024-06-01T02:16:22.835 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph-authtool: AQB2hFpm1g++LxAA96MpkD5ruwF3HnG6V/oCQw== 2024-06-01T02:16:25.943 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph-authtool: AQB5hFpmN0GiDRAAmaPX3ePPHq1DVm0LT+3OaQ== 2024-06-01T02:16:34.310 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph-authtool: AQCBhFpmABhENxAA8gNHainWL4CxCXwjoNPiUQ== 2024-06-01T02:16:34.775 INFO:teuthology.orchestra.run.smithi003.stderr:Creating initial monmap... 2024-06-01T02:16:35.190 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-06-01T02:16:35.190 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-06-01T02:16:35.191 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:16:35.191 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-06-01T02:16:35.621 INFO:teuthology.orchestra.run.smithi003.stderr:monmaptool for smithi003 [v2:172.21.15.3:3300,v1:172.21.15.3:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-06-01T02:16:35.621 INFO:teuthology.orchestra.run.smithi003.stderr:setting min_mon_release = octopus 2024-06-01T02:16:35.621 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/monmaptool: set fsid to e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:16:35.621 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-06-01T02:16:35.621 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:16:35.621 INFO:teuthology.orchestra.run.smithi003.stderr:Creating mon... 2024-06-01T02:16:36.545 INFO:teuthology.orchestra.run.smithi003.stderr:create mon.smithi003 on 2024-06-01T02:16:36.769 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-06-01T02:16:37.008 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968.target → /etc/systemd/system/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968.target. 2024-06-01T02:16:37.008 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968.target → /etc/systemd/system/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968.target. 2024-06-01T02:16:37.330 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: Failed to reset failed state of unit ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi003.service: Unit ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi003.service not loaded. 2024-06-01T02:16:37.337 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: Created symlink /etc/systemd/system/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968.target.wants/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi003.service → /etc/systemd/system/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@.service. 2024-06-01T02:16:38.009 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: disabled 2024-06-01T02:16:38.016 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: inactive 2024-06-01T02:16:38.017 INFO:teuthology.orchestra.run.smithi003.stderr:firewalld.service is not enabled 2024-06-01T02:16:38.017 INFO:teuthology.orchestra.run.smithi003.stderr:Not possible to enable service . firewalld.service is not available 2024-06-01T02:16:38.017 INFO:teuthology.orchestra.run.smithi003.stderr:Waiting for mon to start... 2024-06-01T02:16:38.017 INFO:teuthology.orchestra.run.smithi003.stderr:Waiting for mon... 2024-06-01T02:16:38.901 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: cluster: 2024-06-01T02:16:38.901 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: id: e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:16:38.901 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-06-01T02:16:38.901 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: services: 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi003 (age 0.929232s) 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mgr: no daemons active 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: data: 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: pgs: 2024-06-01T02:16:38.902 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:40.178 INFO:teuthology.orchestra.run.smithi003.stderr:mon is available 2024-06-01T02:16:40.178 INFO:teuthology.orchestra.run.smithi003.stderr:Assimilating anything we can from ceph.conf... 2024-06-01T02:16:40.720 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:40.720 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: [global] 2024-06-01T02:16:40.720 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: fsid = e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:16:40.720 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.3:3300,v1:172.21.15.3:6789] 2024-06-01T02:16:40.720 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: [mgr] 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: [osd] 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-06-01T02:16:40.721 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-06-01T02:16:40.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:40 smithi003 ceph-mon[24782]: from='client.? 172.21.15.3:0/4064439854' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-06-01T02:16:41.267 INFO:teuthology.orchestra.run.smithi003.stderr:Generating new minimal ceph.conf... 2024-06-01T02:16:42.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:41 smithi003 ceph-mon[24782]: from='client.? 172.21.15.3:0/171897309' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:16:42.340 INFO:teuthology.orchestra.run.smithi003.stderr:Restarting the monitor... 2024-06-01T02:16:42.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:42 smithi003 systemd[1]: Stopping Ceph mon.smithi003 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:16:42.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:42 smithi003 bash[25001]: Error: no container with name or ID "ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon.smithi003" found: no such container 2024-06-01T02:16:42.984 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:42 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003[24778]: 2024-06-01T02:16:42.637+0000 7faeb0149700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi003 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:16:42.984 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:42 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003[24778]: 2024-06-01T02:16:42.637+0000 7faeb0149700 -1 mon.smithi003@0(leader) e1 *** Got Signal Terminated *** 2024-06-01T02:16:42.984 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:42 smithi003 podman[25012]: 2024-06-01 02:16:42.709828187 +0000 UTC m=+0.306390904 container died 07d2d4246fe02751cd634b389e91ef846de2366dd1b9721874c652cbed75d461 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.expose-services=, RELEASE=HEAD, name=centos-stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vendor=Red Hat, Inc., distribution-scope=public, GIT_BRANCH=HEAD, architecture=x86_64, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container) 2024-06-01T02:16:43.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 podman[25012]: 2024-06-01 02:16:43.087024659 +0000 UTC m=+0.683587378 container cleanup 07d2d4246fe02751cd634b389e91ef846de2366dd1b9721874c652cbed75d461 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, GIT_BRANCH=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-06-01T02:16:43.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 bash[25012]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003 2024-06-01T02:16:43.532 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 podman[25024]: 2024-06-01 02:16:43.246348312 +0000 UTC m=+0.528393857 container remove 07d2d4246fe02751cd634b389e91ef846de2366dd1b9721874c652cbed75d461 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_CLEAN=True, vcs-type=git, distribution-scope=public, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, ceph=True, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-06-01T02:16:43.532 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 bash[25039]: Error: no container with name or ID "ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon.smithi003" found: no such container 2024-06-01T02:16:43.532 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi003.service: Deactivated successfully. 2024-06-01T02:16:43.532 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 systemd[1]: Stopped Ceph mon.smithi003 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:16:43.532 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 systemd[1]: Starting Ceph mon.smithi003 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:16:43.827 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 podman[25103]: 2024-06-01 02:16:43.440909481 +0000 UTC m=+0.018742630 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 podman[25103]: 2024-06-01 02:16:43.538511222 +0000 UTC m=+0.116344345 container create 108dda5fe7cc0c23a6888f3c445309bfa86e8a0b0a51ab966affb69af2b19757 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, vendor=Red Hat, Inc., name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, architecture=x86_64, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, maintainer=Guillaume Abrioux ) 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 podman[25103]: 2024-06-01 02:16:43.749453557 +0000 UTC m=+0.327286694 container init 108dda5fe7cc0c23a6888f3c445309bfa86e8a0b0a51ab966affb69af2b19757 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, ceph=True, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD) 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 podman[25103]: 2024-06-01 02:16:43.752841545 +0000 UTC m=+0.330674676 container start 108dda5fe7cc0c23a6888f3c445309bfa86e8a0b0a51ab966affb69af2b19757 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, name=centos-stream, GIT_BRANCH=HEAD, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, ceph=True, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: set uid:gid to 167:167 (ceph:ceph) 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: pidfile_write: ignore empty --pid-file 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: load: jerasure load: lrc 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: RocksDB version: 6.15.5 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Compile date Apr 18 2022 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: DB SUMMARY 2024-06-01T02:16:43.828 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: DB Session ID: 4MW5VKNL9XJI6GTIA5Y5 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: CURRENT file: CURRENT 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: IDENTITY file: IDENTITY 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi003/store.db dir, Total Num: 1, files: 000008.sst 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi003/store.db: 000010.log size: 82683 ; 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.error_if_exists: 0 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.create_if_missing: 0 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.paranoid_checks: 1 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.env: 0x558e1cec6860 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.fs: Posix File System 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.info_log: 0x558e1f4effa0 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_file_opening_threads: 16 2024-06-01T02:16:43.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.statistics: (nil) 2024-06-01T02:16:43.830 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.use_fsync: 0 2024-06-01T02:16:43.830 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_log_file_size: 0 2024-06-01T02:16:43.830 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-01T02:16:43.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-01T02:16:43.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.keep_log_file_num: 1000 2024-06-01T02:16:43.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.recycle_log_file_num: 0 2024-06-01T02:16:43.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.allow_fallocate: 1 2024-06-01T02:16:43.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.allow_mmap_reads: 0 2024-06-01T02:16:43.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.allow_mmap_writes: 0 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.use_direct_reads: 0 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.create_missing_column_families: 0 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.db_log_dir: 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi003/store.db 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.advise_random_on_open: 1 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.db_write_buffer_size: 0 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.write_buffer_manager: 0x558e1f5e0330 2024-06-01T02:16:43.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.rate_limiter: (nil) 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.wal_recovery_mode: 2 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.enable_thread_tracking: 0 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.enable_pipelined_write: 0 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.unordered_write: 0 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.row_cache: None 2024-06-01T02:16:43.833 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.wal_filter: None 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.allow_ingest_behind: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.preserve_deletes: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.two_write_queues: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.manual_wal_flush: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.atomic_flush: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.log_readahead_size: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.best_efforts_recovery: 0 2024-06-01T02:16:43.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.allow_data_in_errors: 0 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.db_host_id: __hostname__ 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_background_jobs: 2 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_background_compactions: -1 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_subcompactions: 1 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_total_wal_size: 0 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-01T02:16:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_open_files: -1 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bytes_per_sync: 0 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_readahead_size: 0 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_background_flushes: -1 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Compression algorithms supported: 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: kZSTD supported: 0 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: kXpressCompression supported: 0 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: kLZ4HCCompression supported: 1 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: kLZ4Compression supported: 1 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: kBZip2Compression supported: 0 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: kZlibCompression supported: 1 2024-06-01T02:16:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: kSnappyCompression supported: 1 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi003/store.db/MANIFEST-000009 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.merge_operator: 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_filter: None 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_filter_factory: None 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.sst_partitioner_factory: None 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558e1f4bbd00) 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: cache_index_and_filter_blocks: 1 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-01T02:16:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: pin_top_level_index_and_filter: 1 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: index_type: 0 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: data_block_index_type: 0 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: index_shortening: 1 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: hash_index_allow_collision: 1 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: checksum: 1 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: no_block_cache: 0 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_cache: 0x558e1f526170 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_cache_name: BinnedLRUCache 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_cache_options: 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: capacity : 536870912 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: num_shard_bits : 4 2024-06-01T02:16:43.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: strict_capacity_limit : 0 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: high_pri_pool_ratio: 0.000 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_cache_compressed: (nil) 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: persistent_cache: (nil) 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_size: 4096 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_size_deviation: 10 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_restart_interval: 16 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: index_block_restart_interval: 1 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: metadata_block_size: 4096 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: partition_filters: 0 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: use_delta_encoding: 1 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: whole_key_filtering: 1 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: verify_compression: 0 2024-06-01T02:16:43.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: read_amp_bytes_per_bit: 0 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: format_version: 4 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: enable_index_compression: 1 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_align: 0 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.write_buffer_size: 33554432 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_write_buffer_number: 2 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compression: NoCompression 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bottommost_compression: Disabled 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.prefix_extractor: nullptr 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.num_levels: 7 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-01T02:16:43.840 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compression_opts.level: 32767 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compression_opts.strategy: 0 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compression_opts.enabled: false 2024-06-01T02:16:43.841 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.target_file_size_base: 67108864 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-01T02:16:43.842 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.arena_block_size: 4194304 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.disable_auto_compactions: 0 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-01T02:16:43.843 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.table_properties_collectors: 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.inplace_update_support: 0 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.bloom_locality: 0 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.max_successive_merges: 0 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.paranoid_file_checks: 0 2024-06-01T02:16:43.844 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.force_consistency_checks: 1 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.report_bg_io_stats: 0 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.ttl: 2592000 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.enable_blob_files: false 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.min_blob_size: 0 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.blob_file_size: 268435456 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi003/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717208203780826, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-06-01T02:16:43.845 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717208203781648, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79365, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77629, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71938, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1717208203, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "fc496501-0003-4d45-a912-5886bd277f32", "db_session_id": "4MW5VKNL9XJI6GTIA5Y5"}} 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717208203782132, "job": 1, "event": "recovery_finished"} 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi003/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x558e1f50ca80 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: DB pointer 0x558e1f51c000 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** DB Stats ** 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-01T02:16:43.846 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** Compaction Stats [default] ** 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: L0 2/0 79.30 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.1 0.00 0.00 1 0.001 0 0 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Sum 2/0 79.30 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.1 0.00 0.00 1 0.001 0 0 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.1 0.00 0.00 1 0.001 0 0 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:16:43.847 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** Compaction Stats [default] ** 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 139.1 0.00 0.00 1 0.001 0 0 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Cumulative compaction: 0.00 GB write, 22.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Interval compaction: 0.00 GB write, 22.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-01T02:16:43.848 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** Compaction Stats [default] ** 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: L0 2/0 79.30 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.1 0.00 0.00 1 0.001 0 0 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Sum 2/0 79.30 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.1 0.00 0.00 1 0.001 0 0 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** Compaction Stats [default] ** 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 139.1 0.00 0.00 1 0.001 0 0 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:16:43.849 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Cumulative compaction: 0.00 GB write, 22.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: starting mon.smithi003 rank 0 at public addrs [v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0] at bind addrs [v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi003 fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mon.smithi003@-1(???) e1 preinit fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mon.smithi003@-1(???).mds e1 new map 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mon.smithi003@-1(???).mds e1 print_map 2024-06-01T02:16:43.850 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: e1 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: legacy client fscid: -1 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: No filesystems configured 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mon.smithi003@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mon.smithi003@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mon.smithi003@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mon.smithi003@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mon.smithi003@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 bash[25103]: 108dda5fe7cc0c23a6888f3c445309bfa86e8a0b0a51ab966affb69af2b19757 2024-06-01T02:16:43.851 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 systemd[1]: Started Ceph mon.smithi003 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:16:43.852 INFO:teuthology.orchestra.run.smithi003.stderr:Setting mon public_network to 172.21.0.0/20 2024-06-01T02:16:44.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mon.smithi003 is new leader, mons smithi003 in quorum (ranks 0) 2024-06-01T02:16:44.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: monmap e1: 1 mons at {smithi003=[v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0]} 2024-06-01T02:16:44.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: fsmap 2024-06-01T02:16:44.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: osdmap e1: 0 total, 0 up, 0 in 2024-06-01T02:16:44.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:43 smithi003 ceph-mon[25122]: mgrmap e1: no daemons active 2024-06-01T02:16:44.944 INFO:teuthology.orchestra.run.smithi003.stderr:Wrote config to /etc/ceph/ceph.conf 2024-06-01T02:16:44.945 INFO:teuthology.orchestra.run.smithi003.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-06-01T02:16:44.945 INFO:teuthology.orchestra.run.smithi003.stderr:Creating mgr... 2024-06-01T02:16:44.945 INFO:teuthology.orchestra.run.smithi003.stderr:Verifying port 9283 ... 2024-06-01T02:16:45.189 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: Failed to reset failed state of unit ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mgr.smithi003.njsqxz.service: Unit ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mgr.smithi003.njsqxz.service not loaded. 2024-06-01T02:16:45.197 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: Created symlink /etc/systemd/system/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968.target.wants/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mgr.smithi003.njsqxz.service → /etc/systemd/system/ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@.service. 2024-06-01T02:16:45.698 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:45 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1782127784' entity='client.admin' 2024-06-01T02:16:45.910 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: disabled 2024-06-01T02:16:45.919 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: inactive 2024-06-01T02:16:45.919 INFO:teuthology.orchestra.run.smithi003.stderr:firewalld.service is not enabled 2024-06-01T02:16:45.919 INFO:teuthology.orchestra.run.smithi003.stderr:Not possible to enable service . firewalld.service is not available 2024-06-01T02:16:45.928 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: disabled 2024-06-01T02:16:45.935 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: inactive 2024-06-01T02:16:45.937 INFO:teuthology.orchestra.run.smithi003.stderr:firewalld.service is not enabled 2024-06-01T02:16:45.937 INFO:teuthology.orchestra.run.smithi003.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-06-01T02:16:45.937 INFO:teuthology.orchestra.run.smithi003.stderr:Waiting for mgr to start... 2024-06-01T02:16:45.937 INFO:teuthology.orchestra.run.smithi003.stderr:Waiting for mgr... 2024-06-01T02:16:46.446 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:46.446 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: { 2024-06-01T02:16:46.446 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "fsid": "e42641dc-1fbc-11ef-bc9b-c7b262605968", 2024-06-01T02:16:46.446 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "health": { 2024-06-01T02:16:46.446 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-01T02:16:46.446 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "checks": {}, 2024-06-01T02:16:46.446 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "mutes": [] 2024-06-01T02:16:46.446 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:46.446 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "quorum": [ 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 0 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ], 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "smithi003" 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ], 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "monmap": { 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "osdmap": { 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:46.447 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "pgmap": { 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-01T02:16:46.448 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "fsmap": { 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "available": false, 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "modules": [ 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "iostat", 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "nfs", 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "restful" 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ], 2024-06-01T02:16:46.449 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "services": {} 2024-06-01T02:16:46.450 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:46.450 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "servicemap": { 2024-06-01T02:16:46.450 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:46.450 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "modified": "2024-06-01T02:16:37.976494+0000", 2024-06-01T02:16:46.450 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "services": {} 2024-06-01T02:16:46.450 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:46.450 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-01T02:16:46.450 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: } 2024-06-01T02:16:46.557 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:46 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/718722640' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-01T02:16:47.030 INFO:teuthology.orchestra.run.smithi003.stderr:mgr not available, waiting (1/15)... 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: { 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "fsid": "e42641dc-1fbc-11ef-bc9b-c7b262605968", 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "health": { 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "checks": {}, 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "mutes": [] 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "quorum": [ 2024-06-01T02:16:50.011 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 0 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ], 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "smithi003" 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ], 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "monmap": { 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "osdmap": { 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-01T02:16:50.012 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-01T02:16:50.013 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-01T02:16:50.013 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-01T02:16:50.016 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-01T02:16:50.016 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-01T02:16:50.016 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:50.016 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "pgmap": { 2024-06-01T02:16:50.016 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-01T02:16:50.016 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "fsmap": { 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:50.017 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "available": false, 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "modules": [ 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "iostat", 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "nfs", 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "restful" 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ], 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "services": {} 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "servicemap": { 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "modified": "2024-06-01T02:16:37.976494+0000", 2024-06-01T02:16:50.018 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "services": {} 2024-06-01T02:16:50.019 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:50.019 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-01T02:16:50.019 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: } 2024-06-01T02:16:50.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:50 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2883420386' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-01T02:16:50.594 INFO:teuthology.orchestra.run.smithi003.stderr:mgr not available, waiting (2/15)... 2024-06-01T02:16:52.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: Activating manager daemon smithi003.njsqxz 2024-06-01T02:16:52.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: mgrmap e2: smithi003.njsqxz(active, starting, since 0.00273886s) 2024-06-01T02:16:52.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:16:52.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:16:52.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:16:52.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:16:52.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:16:52.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: Manager daemon smithi003.njsqxz is now available 2024-06-01T02:16:52.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:16:52.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:16:52.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' 2024-06-01T02:16:52.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' 2024-06-01T02:16:52.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:52 smithi003 ceph-mon[25122]: from='mgr.14100 172.21.15.3:0/2015197415' entity='mgr.smithi003.njsqxz' 2024-06-01T02:16:53.160 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:53.160 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: { 2024-06-01T02:16:53.160 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "fsid": "e42641dc-1fbc-11ef-bc9b-c7b262605968", 2024-06-01T02:16:53.160 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "health": { 2024-06-01T02:16:53.160 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-01T02:16:53.160 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "checks": {}, 2024-06-01T02:16:53.160 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "mutes": [] 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "quorum": [ 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 0 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ], 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "smithi003" 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ], 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "monmap": { 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-01T02:16:53.161 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:53.162 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "osdmap": { 2024-06-01T02:16:53.162 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:53.162 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-01T02:16:53.162 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-01T02:16:53.162 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-01T02:16:53.162 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-01T02:16:53.165 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-01T02:16:53.165 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-01T02:16:53.165 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:53.165 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "pgmap": { 2024-06-01T02:16:53.165 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "fsmap": { 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-01T02:16:53.166 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "available": true, 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "modules": [ 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "iostat", 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "nfs", 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "restful" 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ], 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "services": {} 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "servicemap": { 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "modified": "2024-06-01T02:16:37.976494+0000", 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "services": {} 2024-06-01T02:16:53.167 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: }, 2024-06-01T02:16:53.168 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-01T02:16:53.168 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: } 2024-06-01T02:16:53.690 INFO:teuthology.orchestra.run.smithi003.stderr:mgr is available 2024-06-01T02:16:54.325 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:54.325 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: [global] 2024-06-01T02:16:54.326 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: fsid = e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:16:54.326 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-06-01T02:16:54.326 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-06-01T02:16:54.326 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-06-01T02:16:54.326 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-06-01T02:16:54.326 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:54.327 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: [mgr] 2024-06-01T02:16:54.327 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-06-01T02:16:54.327 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 2024-06-01T02:16:54.327 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: [osd] 2024-06-01T02:16:54.327 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-06-01T02:16:54.327 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-06-01T02:16:54.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:54 smithi003 ceph-mon[25122]: mgrmap e3: smithi003.njsqxz(active, since 1.00614s) 2024-06-01T02:16:54.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:54 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2627865141' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-01T02:16:54.877 INFO:teuthology.orchestra.run.smithi003.stderr:Enabling cephadm module... 2024-06-01T02:16:55.261 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:55 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2005889491' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-06-01T02:16:55.261 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:55 smithi003 ceph-mon[25122]: mgrmap e4: smithi003.njsqxz(active, since 2s) 2024-06-01T02:16:56.405 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:56 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1915131327' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-06-01T02:16:57.281 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:57 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1915131327' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-06-01T02:16:57.281 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:57 smithi003 ceph-mon[25122]: mgrmap e5: smithi003.njsqxz(active, since 4s) 2024-06-01T02:16:57.282 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: { 2024-06-01T02:16:57.282 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 5, 2024-06-01T02:16:57.282 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "available": true, 2024-06-01T02:16:57.282 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "active_name": "smithi003.njsqxz", 2024-06-01T02:16:57.283 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_standby": 0 2024-06-01T02:16:57.283 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: } 2024-06-01T02:16:57.876 INFO:teuthology.orchestra.run.smithi003.stderr:Waiting for the mgr to restart... 2024-06-01T02:16:57.876 INFO:teuthology.orchestra.run.smithi003.stderr:Waiting for mgr epoch 5... 2024-06-01T02:16:58.274 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:16:58 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/123988142' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-06-01T02:17:03.431 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: { 2024-06-01T02:17:03.431 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-06-01T02:17:03.431 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "initialized": true 2024-06-01T02:17:03.431 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: } 2024-06-01T02:17:03.723 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: Active manager daemon smithi003.njsqxz restarted 2024-06-01T02:17:03.723 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: Activating manager daemon smithi003.njsqxz 2024-06-01T02:17:03.723 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: osdmap e2: 0 total, 0 up, 0 in 2024-06-01T02:17:03.723 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: mgrmap e6: smithi003.njsqxz(active, starting, since 0.0527022s) 2024-06-01T02:17:03.723 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:17:03.723 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:17:03.723 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:17:03.723 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:17:03.724 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:17:03.724 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: Manager daemon smithi003.njsqxz is now available 2024-06-01T02:17:03.724 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:03.724 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:03.724 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:17:03.724 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:17:03.724 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:17:03.724 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:17:03.724 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:03 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:17:03.992 INFO:teuthology.orchestra.run.smithi003.stderr:mgr epoch 5 is available 2024-06-01T02:17:03.992 INFO:teuthology.orchestra.run.smithi003.stderr:Setting orchestrator backend to cephadm... 2024-06-01T02:17:04.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:04 smithi003 ceph-mon[25122]: mgrmap e7: smithi003.njsqxz(active, since 1.05516s) 2024-06-01T02:17:04.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:04 smithi003 ceph-mon[25122]: [01/Jun/2024:02:17:03] ENGINE Bus STARTING 2024-06-01T02:17:04.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:04 smithi003 ceph-mon[25122]: [01/Jun/2024:02:17:03] ENGINE Serving on https://172.21.15.3:7150 2024-06-01T02:17:04.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:04 smithi003 ceph-mon[25122]: [01/Jun/2024:02:17:03] ENGINE Bus STARTED 2024-06-01T02:17:04.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:04 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:04.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:04 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:17:05.699 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:05 smithi003 ceph-mon[25122]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:05.699 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:05 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:05.699 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:05 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:17:05.699 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:05 smithi003 ceph-mon[25122]: mgrmap e8: smithi003.njsqxz(active, since 2s) 2024-06-01T02:17:05.903 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: value unchanged 2024-06-01T02:17:06.483 INFO:teuthology.orchestra.run.smithi003.stderr:Generating ssh key... 2024-06-01T02:17:06.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:06 smithi003 ceph-mon[25122]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:07.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:07 smithi003 ceph-mon[25122]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:07.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:07 smithi003 ceph-mon[25122]: Generating ssh key... 2024-06-01T02:17:07.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:07 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:07.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:07 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:08.672 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbNtBBoKc7E64WJ1ac/7YK/vrO3bVZ8h5U2jz2dnHcD7jpu4ZKIR8cyLwEE2lk4/dM7iu3J/TQq0CV6INpFnbdt0tqHUoeHu7O9ja62AI4fmtw8KY16YtjVGD0XojxC1yQLn2XLhvDzOYWkRreaYw1h8FXx+TWW/Zb9DOnU9HsWVbd5RqR90rIS8VYFqUuA7h7de10EYF/QY/ilA44Y8Di+cl0jZw6CIWAJylIvVS/S4g8eqYakcOlupbHYDrofphK2nw9cs69T35bPQAyVGEL8vl0Q4s8d50MsfsszlyMcu7x+7hxjgHFENeo6QQ4D74PRfjGI8A56HRMoEKB4mLcnAzIdTmQvUKsawWJdh/Gv3rmBfxVxiVkDzasnKZa+g3gh3q9/gTGPQdm9KzXxqKcLR2kaEWTmSEx0pH4EueXuX5njRSqjFstb6XP5DafBvRkfb9Zv9M8Cfe9xqMZSkTw+jSF4mRGf67+zx7dgHA+YvRQvlLn0AQmGjSHyGd10Xc= ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:17:09.199 INFO:teuthology.orchestra.run.smithi003.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-06-01T02:17:09.199 INFO:teuthology.orchestra.run.smithi003.stderr:Adding key to root@localhost authorized_keys... 2024-06-01T02:17:09.199 INFO:teuthology.orchestra.run.smithi003.stderr:Adding host smithi003... 2024-06-01T02:17:09.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:09 smithi003 ceph-mon[25122]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:10.714 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: Added host 'smithi003' with addr '172.21.15.3' 2024-06-01T02:17:11.028 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:10 smithi003 ceph-mon[25122]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi003", "addr": "172.21.15.3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:11.028 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:10 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:11.028 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:10 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:17:11.305 INFO:teuthology.orchestra.run.smithi003.stderr:Deploying mon service with default placement... 2024-06-01T02:17:11.971 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:11 smithi003 ceph-mon[25122]: Deploying cephadm binary to smithi003 2024-06-01T02:17:11.972 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:11 smithi003 ceph-mon[25122]: Added host smithi003 2024-06-01T02:17:11.972 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: Scheduled mon update... 2024-06-01T02:17:12.663 INFO:teuthology.orchestra.run.smithi003.stderr:Deploying mgr service with default placement... 2024-06-01T02:17:13.289 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:12 smithi003 ceph-mon[25122]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:13.289 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:12 smithi003 ceph-mon[25122]: Saving service mon spec with placement count:5 2024-06-01T02:17:13.290 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:12 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:13.392 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-06-01T02:17:14.693 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:14 smithi003 ceph-mon[25122]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:14.693 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:14 smithi003 ceph-mon[25122]: Saving service mgr spec with placement count:2 2024-06-01T02:17:14.693 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:14 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:14.693 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:14 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:14.695 INFO:teuthology.orchestra.run.smithi003.stderr:Deploying crash service with default placement... 2024-06-01T02:17:15.342 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: Scheduled crash update... 2024-06-01T02:17:16.018 INFO:teuthology.orchestra.run.smithi003.stderr:Deploying prometheus service with default placement... 2024-06-01T02:17:16.246 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:15 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:16.246 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:15 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:16.677 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-06-01T02:17:17.270 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:16 smithi003 ceph-mon[25122]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:17.270 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:16 smithi003 ceph-mon[25122]: Saving service crash spec with placement * 2024-06-01T02:17:17.270 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:16 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:17.273 INFO:teuthology.orchestra.run.smithi003.stderr:Deploying grafana service with default placement... 2024-06-01T02:17:17.931 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-06-01T02:17:18.243 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:17 smithi003 ceph-mon[25122]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:18.244 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:17 smithi003 ceph-mon[25122]: Saving service prometheus spec with placement count:1 2024-06-01T02:17:18.244 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:17 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:18.244 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:17 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:18.513 INFO:teuthology.orchestra.run.smithi003.stderr:Deploying node-exporter service with default placement... 2024-06-01T02:17:19.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:18 smithi003 ceph-mon[25122]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:19.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:18 smithi003 ceph-mon[25122]: Saving service grafana spec with placement count:1 2024-06-01T02:17:19.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:18 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:19.187 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-06-01T02:17:19.713 INFO:teuthology.orchestra.run.smithi003.stderr:Deploying alertmanager service with default placement... 2024-06-01T02:17:20.256 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:20 smithi003 ceph-mon[25122]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:20.256 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:20 smithi003 ceph-mon[25122]: Saving service node-exporter spec with placement * 2024-06-01T02:17:20.256 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:20 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:20.269 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-06-01T02:17:21.470 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:21 smithi003 ceph-mon[25122]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:21.471 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:21 smithi003 ceph-mon[25122]: Saving service alertmanager spec with placement count:1 2024-06-01T02:17:21.471 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:21 smithi003 ceph-mon[25122]: from='mgr.14118 172.21.15.3:0/2393744174' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:22.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:22 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1975393374' entity='client.admin' 2024-06-01T02:17:23.791 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:23 smithi003 ceph-mon[25122]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:17:23.791 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:23 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3224184417' entity='client.admin' 2024-06-01T02:17:23.794 INFO:teuthology.orchestra.run.smithi003.stderr:Enabling the dashboard module... 2024-06-01T02:17:25.080 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:24 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1413906881' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-06-01T02:17:26.007 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:25 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1413906881' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-06-01T02:17:26.007 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:25 smithi003 ceph-mon[25122]: mgrmap e9: smithi003.njsqxz(active, since 22s) 2024-06-01T02:17:26.008 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: { 2024-06-01T02:17:26.008 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "epoch": 9, 2024-06-01T02:17:26.008 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "available": true, 2024-06-01T02:17:26.008 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "active_name": "smithi003.njsqxz", 2024-06-01T02:17:26.008 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "num_standby": 0 2024-06-01T02:17:26.008 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: } 2024-06-01T02:17:26.641 INFO:teuthology.orchestra.run.smithi003.stderr:Waiting for the mgr to restart... 2024-06-01T02:17:26.641 INFO:teuthology.orchestra.run.smithi003.stderr:Waiting for mgr epoch 9... 2024-06-01T02:17:26.893 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:26 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2815276511' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-06-01T02:17:31.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:31 smithi003 ceph-mon[25122]: Active manager daemon smithi003.njsqxz restarted 2024-06-01T02:17:31.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:31 smithi003 ceph-mon[25122]: Activating manager daemon smithi003.njsqxz 2024-06-01T02:17:31.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:31 smithi003 ceph-mon[25122]: osdmap e3: 0 total, 0 up, 0 in 2024-06-01T02:17:32.163 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: { 2024-06-01T02:17:32.163 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-06-01T02:17:32.163 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: "initialized": true 2024-06-01T02:17:32.163 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: } 2024-06-01T02:17:32.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: mgrmap e10: smithi003.njsqxz(active, starting, since 0.0551446s) 2024-06-01T02:17:32.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:17:32.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:17:32.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:17:32.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:17:32.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:17:32.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: Manager daemon smithi003.njsqxz is now available 2024-06-01T02:17:32.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:17:32.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:17:32.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:17:32.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:17:32.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: [01/Jun/2024:02:17:31] ENGINE Bus STARTING 2024-06-01T02:17:32.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:32 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:32.868 INFO:teuthology.orchestra.run.smithi003.stderr:mgr epoch 9 is available 2024-06-01T02:17:32.868 INFO:teuthology.orchestra.run.smithi003.stderr:Generating a dashboard self-signed certificate... 2024-06-01T02:17:33.399 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:33 smithi003 ceph-mon[25122]: [01/Jun/2024:02:17:31] ENGINE Serving on https://172.21.15.3:7150 2024-06-01T02:17:33.399 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:33 smithi003 ceph-mon[25122]: [01/Jun/2024:02:17:31] ENGINE Bus STARTED 2024-06-01T02:17:33.399 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:33 smithi003 ceph-mon[25122]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-06-01T02:17:33.399 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:33 smithi003 ceph-mon[25122]: mgrmap e11: smithi003.njsqxz(active, since 1.06538s) 2024-06-01T02:17:33.399 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:33 smithi003 ceph-mon[25122]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-06-01T02:17:33.622 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: Self-signed certificate created 2024-06-01T02:17:34.264 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:34 smithi003 ceph-mon[25122]: mgrmap e12: smithi003.njsqxz(active, since 2s) 2024-06-01T02:17:34.264 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:34 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:34.264 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:34 smithi003 ceph-mon[25122]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:34.264 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:34 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:34.264 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:34 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:34.266 INFO:teuthology.orchestra.run.smithi003.stderr:Creating initial admin user... 2024-06-01T02:17:35.291 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$Zm5ejXOjIFxW.PsI6.jWaeTlqnzZZYHDFA.oMgBWz5hs5SrGcEzuW", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1717208255, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-06-01T02:17:35.577 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:35 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:35.577 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:35 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:36.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:36 smithi003 ceph-mon[25122]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:36.563 INFO:teuthology.orchestra.run.smithi003.stderr:Fetching dashboard port number... 2024-06-01T02:17:37.197 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: 8443 2024-06-01T02:17:37.525 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:37 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2361340903' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-06-01T02:17:37.830 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: disabled 2024-06-01T02:17:37.838 INFO:teuthology.orchestra.run.smithi003.stderr:systemctl: inactive 2024-06-01T02:17:37.839 INFO:teuthology.orchestra.run.smithi003.stderr:firewalld.service is not enabled 2024-06-01T02:17:37.839 INFO:teuthology.orchestra.run.smithi003.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-06-01T02:17:37.842 INFO:teuthology.orchestra.run.smithi003.stderr:Ceph Dashboard is now available at: 2024-06-01T02:17:37.842 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:17:37.842 INFO:teuthology.orchestra.run.smithi003.stderr: URL: https://smithi003.front.sepia.ceph.com:8443/ 2024-06-01T02:17:37.842 INFO:teuthology.orchestra.run.smithi003.stderr: User: admin 2024-06-01T02:17:37.842 INFO:teuthology.orchestra.run.smithi003.stderr: Password: iw452fnnzv 2024-06-01T02:17:37.842 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:17:37.842 INFO:teuthology.orchestra.run.smithi003.stderr:Enabling autotune for osd_memory_target 2024-06-01T02:17:38.429 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:38 smithi003 ceph-mon[25122]: mgrmap e13: smithi003.njsqxz(active, since 6s) 2024-06-01T02:17:39.944 INFO:teuthology.orchestra.run.smithi003.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-06-01T02:17:40.534 INFO:teuthology.orchestra.run.smithi003.stderr:You can access the Ceph CLI with: 2024-06-01T02:17:40.534 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:17:40.534 INFO:teuthology.orchestra.run.smithi003.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr:Please consider enabling telemetry to help improve Ceph: 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr: ceph telemetry on 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr:For more information see: 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:17:40.535 INFO:teuthology.orchestra.run.smithi003.stderr:Bootstrap complete. 2024-06-01T02:17:40.578 INFO:tasks.cephadm:Fetching config... 2024-06-01T02:17:40.578 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:17:40.578 DEBUG:teuthology.orchestra.run.smithi003:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-06-01T02:17:40.596 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-06-01T02:17:40.596 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:17:40.596 DEBUG:teuthology.orchestra.run.smithi003:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-06-01T02:17:40.653 INFO:tasks.cephadm:Fetching mon keyring... 2024-06-01T02:17:40.653 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:17:40.653 DEBUG:teuthology.orchestra.run.smithi003:> sudo dd if=/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/keyring of=/dev/stdout 2024-06-01T02:17:40.721 INFO:tasks.cephadm:Fetching pub ssh key... 2024-06-01T02:17:40.721 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:17:40.722 DEBUG:teuthology.orchestra.run.smithi003:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-06-01T02:17:40.778 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-06-01T02:17:40.778 DEBUG:teuthology.orchestra.run.smithi003:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbNtBBoKc7E64WJ1ac/7YK/vrO3bVZ8h5U2jz2dnHcD7jpu4ZKIR8cyLwEE2lk4/dM7iu3J/TQq0CV6INpFnbdt0tqHUoeHu7O9ja62AI4fmtw8KY16YtjVGD0XojxC1yQLn2XLhvDzOYWkRreaYw1h8FXx+TWW/Zb9DOnU9HsWVbd5RqR90rIS8VYFqUuA7h7de10EYF/QY/ilA44Y8Di+cl0jZw6CIWAJylIvVS/S4g8eqYakcOlupbHYDrofphK2nw9cs69T35bPQAyVGEL8vl0Q4s8d50MsfsszlyMcu7x+7hxjgHFENeo6QQ4D74PRfjGI8A56HRMoEKB4mLcnAzIdTmQvUKsawWJdh/Gv3rmBfxVxiVkDzasnKZa+g3gh3q9/gTGPQdm9KzXxqKcLR2kaEWTmSEx0pH4EueXuX5njRSqjFstb6XP5DafBvRkfb9Zv9M8Cfe9xqMZSkTw+jSF4mRGf67+zx7dgHA+YvRQvlLn0AQmGjSHyGd10Xc= ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-06-01T02:17:40.858 INFO:teuthology.orchestra.run.smithi003.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbNtBBoKc7E64WJ1ac/7YK/vrO3bVZ8h5U2jz2dnHcD7jpu4ZKIR8cyLwEE2lk4/dM7iu3J/TQq0CV6INpFnbdt0tqHUoeHu7O9ja62AI4fmtw8KY16YtjVGD0XojxC1yQLn2XLhvDzOYWkRreaYw1h8FXx+TWW/Zb9DOnU9HsWVbd5RqR90rIS8VYFqUuA7h7de10EYF/QY/ilA44Y8Di+cl0jZw6CIWAJylIvVS/S4g8eqYakcOlupbHYDrofphK2nw9cs69T35bPQAyVGEL8vl0Q4s8d50MsfsszlyMcu7x+7hxjgHFENeo6QQ4D74PRfjGI8A56HRMoEKB4mLcnAzIdTmQvUKsawWJdh/Gv3rmBfxVxiVkDzasnKZa+g3gh3q9/gTGPQdm9KzXxqKcLR2kaEWTmSEx0pH4EueXuX5njRSqjFstb6XP5DafBvRkfb9Zv9M8Cfe9xqMZSkTw+jSF4mRGf67+zx7dgHA+YvRQvlLn0AQmGjSHyGd10Xc= ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:17:40.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbNtBBoKc7E64WJ1ac/7YK/vrO3bVZ8h5U2jz2dnHcD7jpu4ZKIR8cyLwEE2lk4/dM7iu3J/TQq0CV6INpFnbdt0tqHUoeHu7O9ja62AI4fmtw8KY16YtjVGD0XojxC1yQLn2XLhvDzOYWkRreaYw1h8FXx+TWW/Zb9DOnU9HsWVbd5RqR90rIS8VYFqUuA7h7de10EYF/QY/ilA44Y8Di+cl0jZw6CIWAJylIvVS/S4g8eqYakcOlupbHYDrofphK2nw9cs69T35bPQAyVGEL8vl0Q4s8d50MsfsszlyMcu7x+7hxjgHFENeo6QQ4D74PRfjGI8A56HRMoEKB4mLcnAzIdTmQvUKsawWJdh/Gv3rmBfxVxiVkDzasnKZa+g3gh3q9/gTGPQdm9KzXxqKcLR2kaEWTmSEx0pH4EueXuX5njRSqjFstb6XP5DafBvRkfb9Zv9M8Cfe9xqMZSkTw+jSF4mRGf67+zx7dgHA+YvRQvlLn0AQmGjSHyGd10Xc= ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-06-01T02:17:40.914 INFO:teuthology.orchestra.run.smithi027.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDbNtBBoKc7E64WJ1ac/7YK/vrO3bVZ8h5U2jz2dnHcD7jpu4ZKIR8cyLwEE2lk4/dM7iu3J/TQq0CV6INpFnbdt0tqHUoeHu7O9ja62AI4fmtw8KY16YtjVGD0XojxC1yQLn2XLhvDzOYWkRreaYw1h8FXx+TWW/Zb9DOnU9HsWVbd5RqR90rIS8VYFqUuA7h7de10EYF/QY/ilA44Y8Di+cl0jZw6CIWAJylIvVS/S4g8eqYakcOlupbHYDrofphK2nw9cs69T35bPQAyVGEL8vl0Q4s8d50MsfsszlyMcu7x+7hxjgHFENeo6QQ4D74PRfjGI8A56HRMoEKB4mLcnAzIdTmQvUKsawWJdh/Gv3rmBfxVxiVkDzasnKZa+g3gh3q9/gTGPQdm9KzXxqKcLR2kaEWTmSEx0pH4EueXuX5njRSqjFstb6XP5DafBvRkfb9Zv9M8Cfe9xqMZSkTw+jSF4mRGf67+zx7dgHA+YvRQvlLn0AQmGjSHyGd10Xc= ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:17:40.927 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-06-01T02:17:41.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:40 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1052971465' entity='client.admin' 2024-06-01T02:17:44.129 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-06-01T02:17:44.130 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-06-01T02:17:44.471 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:44 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:44.472 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:44 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd/host:smithi003", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:17:44.472 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:44 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:44.472 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:44 smithi003 ceph-mon[25122]: Deploying daemon alertmanager.smithi003 on smithi003 2024-06-01T02:17:44.472 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:44 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/558875085' entity='client.admin' 2024-06-01T02:17:47.678 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi027 2024-06-01T02:17:47.679 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-01T02:17:47.679 DEBUG:teuthology.orchestra.run.smithi027:> dd of=/etc/ceph/ceph.conf 2024-06-01T02:17:47.698 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-01T02:17:47.698 DEBUG:teuthology.orchestra.run.smithi027:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:17:47.757 INFO:tasks.cephadm:Adding host smithi027 to orchestrator... 2024-06-01T02:17:47.757 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch host add smithi027 2024-06-01T02:17:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:47 smithi003 ceph-mon[25122]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:47 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:52.069 INFO:teuthology.orchestra.run.smithi003.stdout:Added host 'smithi027' with addr '172.21.15.27' 2024-06-01T02:17:52.511 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:52 smithi003 ceph-mon[25122]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi027", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:17:52.511 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:52 smithi003 ceph-mon[25122]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:17:52.511 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:52 smithi003 ceph-mon[25122]: Deploying cephadm binary to smithi027 2024-06-01T02:17:52.511 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:52 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:52.789 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch host ls --format=json 2024-06-01T02:17:53.303 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:53 smithi003 ceph-mon[25122]: Added host smithi027 2024-06-01T02:17:55.060 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:54 smithi003 ceph-mon[25122]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:17:55.983 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:17:55.983 INFO:teuthology.orchestra.run.smithi003.stdout:[{"addr": "172.21.15.3", "hostname": "smithi003", "labels": [], "status": ""}, {"addr": "172.21.15.27", "hostname": "smithi027", "labels": [], "status": ""}] 2024-06-01T02:17:57.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:56 smithi003 ceph-mon[25122]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:17:57.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:56 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:17:57.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:56 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi003", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:17:57.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:56 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi003", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-06-01T02:17:57.083 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:56 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:17:57.293 INFO:tasks.cephadm:Setting crush tunables to default 2024-06-01T02:17:57.293 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd crush tunables default 2024-06-01T02:17:57.818 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:57 smithi003 ceph-mon[25122]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:17:57.818 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:57 smithi003 ceph-mon[25122]: Deploying daemon crash.smithi003 on smithi003 2024-06-01T02:17:59.036 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:17:58 smithi003 ceph-mon[25122]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:00.807 INFO:teuthology.orchestra.run.smithi003.stderr:adjusted tunables profile to default 2024-06-01T02:18:01.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:00 smithi003 ceph-mon[25122]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:01.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:00 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/419914826' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-06-01T02:18:01.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:00 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:01.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:00 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:01.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:00 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:01.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:00 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-01T02:18:01.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:00 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:01.410 INFO:tasks.cephadm:Adding mon.smithi003 on smithi003 2024-06-01T02:18:01.410 INFO:tasks.cephadm:Adding mon.smithi027 on smithi027 2024-06-01T02:18:01.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch apply mon '2;smithi003:172.21.15.3=smithi003;smithi027:172.21.15.27=smithi027' 2024-06-01T02:18:02.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:01 smithi003 ceph-mon[25122]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-01T02:18:02.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:01 smithi003 ceph-mon[25122]: Deploying daemon grafana.smithi003 on smithi003 2024-06-01T02:18:02.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:01 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/419914826' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-06-01T02:18:02.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:01 smithi003 ceph-mon[25122]: osdmap e4: 0 total, 0 up, 0 in 2024-06-01T02:18:02.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:01 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:03.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:02 smithi003 ceph-mon[25122]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:03.186 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled mon update... 2024-06-01T02:18:03.630 DEBUG:teuthology.orchestra.run.smithi027:mon.smithi027> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi027.service 2024-06-01T02:18:03.632 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:03.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:04.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:04 smithi003 ceph-mon[25122]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:04.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:04 smithi003 ceph-mon[25122]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi003:172.21.15.3=smithi003;smithi027:172.21.15.27=smithi027", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:18:04.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:04 smithi003 ceph-mon[25122]: Saving service mon spec with placement smithi003:172.21.15.3=smithi003;smithi027:172.21.15.27=smithi027;count:2 2024-06-01T02:18:04.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:04 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:05.482 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:05.482 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:05.483 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:06.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:06 smithi003 ceph-mon[25122]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:06.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:06 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/4258937747' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:07.053 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:07.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:08.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:08 smithi003 ceph-mon[25122]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:08.632 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:08.632 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:08.634 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:09.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:09 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/3909526768' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:10.195 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:10.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:11.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:10 smithi003 ceph-mon[25122]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:11.895 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:11.895 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:11.897 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:12.931 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:12 smithi003 ceph-mon[25122]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:12.931 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:12 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/3089567829' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:13.434 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:13.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:14.951 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:14 smithi003 ceph-mon[25122]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:14.951 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:14 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:15.376 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:15.376 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:15.378 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:16.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:15 smithi003 ceph-mon[25122]: Deploying daemon node-exporter.smithi003 on smithi003 2024-06-01T02:18:16.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:15 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/3313471893' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:16.954 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:16.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:17.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:16 smithi003 ceph-mon[25122]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:17.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:16 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:18.666 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:18.667 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:18.668 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:18.810 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:18 smithi003 ceph-mon[25122]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:18.810 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:18 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:18.810 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:18 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-01T02:18:19.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:19 smithi003 ceph-mon[25122]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-01T02:18:19.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:19 smithi003 ceph-mon[25122]: mgrmap e14: smithi003.njsqxz(active, since 47s) 2024-06-01T02:18:19.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:19 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/2575397509' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:20.364 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:20.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:22.166 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:22.167 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:22.168 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:22.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:22 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/3861241531' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:23.761 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:23.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:25.608 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:25.608 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:25.609 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:25.948 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: Active manager daemon smithi003.njsqxz restarted 2024-06-01T02:18:25.948 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: Activating manager daemon smithi003.njsqxz 2024-06-01T02:18:25.948 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: osdmap e5: 0 total, 0 up, 0 in 2024-06-01T02:18:25.948 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: mgrmap e15: smithi003.njsqxz(active, starting, since 0.00964875s) 2024-06-01T02:18:25.948 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:18:25.948 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:18:25.948 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:18:25.948 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:18:25.948 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: Manager daemon smithi003.njsqxz is now available 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:25.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:25 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/2944014260' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:27.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:26 smithi003 ceph-mon[25122]: mgrmap e16: smithi003.njsqxz(active, since 1.01267s) 2024-06-01T02:18:27.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:27.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:26 smithi003 ceph-mon[25122]: [01/Jun/2024:02:18:25] ENGINE Bus STARTING 2024-06-01T02:18:27.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:26 smithi003 ceph-mon[25122]: [01/Jun/2024:02:18:26] ENGINE Serving on https://172.21.15.3:7150 2024-06-01T02:18:27.083 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:26 smithi003 ceph-mon[25122]: [01/Jun/2024:02:18:26] ENGINE Bus STARTED 2024-06-01T02:18:27.083 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:27.083 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:27.214 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:27.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:28.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:28 smithi003 ceph-mon[25122]: mgrmap e17: smithi003.njsqxz(active, since 2s) 2024-06-01T02:18:28.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:28 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:28.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:28 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:28.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:28 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd/host:smithi003", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:18:28.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:28 smithi003 ceph-mon[25122]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:18:30.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:30 smithi003 ceph-mon[25122]: Updating smithi003:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:18:30.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:30 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:30.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:30 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:30.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:30 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:30.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:30 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd/host:smithi027", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:18:30.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:30 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:30.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:30 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:18:30.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:30 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-06-01T02:18:30.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:30 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:30.489 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:30.489 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:30.491 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:31.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:31 smithi003 ceph-mon[25122]: Deploying daemon crash.smithi027 on smithi027 2024-06-01T02:18:31.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:31 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/1172041388' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:32.791 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:32.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:34.035 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:34.035 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:18:34.035 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-01T02:18:34.035 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:18:34.035 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:34.035 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:33 smithi003 ceph-mon[25122]: Deploying daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:18:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:34.494 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:34.496 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:35.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:34 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/4275143857' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:36.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:35 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:37.022 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:37.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:38.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:37 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:38.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:37 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:18:38.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:37 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:38.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:37 smithi003 ceph-mon[25122]: Deploying daemon mon.smithi027 on smithi027 2024-06-01T02:18:40.713 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:40.713 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:16:35.108780Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-01T02:18:40.715 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2024-06-01T02:18:40.817 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:40 smithi027 podman[27234]: 2024-06-01 02:18:40.70099371 +0000 UTC m=+0.018896204 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-06-01T02:18:40.817 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:40 smithi027 podman[27234]: 2024-06-01 02:18:40.822364261 +0000 UTC m=+0.140266731 container create edc537c54a3ad9f24f56782b13871c4c220ec52f067649c1356d66c541103ede (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, version=8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-06-01T02:18:41.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:40 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:40 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/2037932566' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.392 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 podman[27234]: 2024-06-01 02:18:41.153114077 +0000 UTC m=+0.471016558 container init edc537c54a3ad9f24f56782b13871c4c220ec52f067649c1356d66c541103ede (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, version=8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-06-01T02:18:41.392 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 podman[27234]: 2024-06-01 02:18:41.15716963 +0000 UTC m=+0.475072112 container start edc537c54a3ad9f24f56782b13871c4c220ec52f067649c1356d66c541103ede (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.component=centos-stream-container, ceph=True, distribution-scope=public, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , version=8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.buildah.version=1.19.8, io.openshift.expose-services=) 2024-06-01T02:18:41.392 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: set uid:gid to 167:167 (ceph:ceph) 2024-06-01T02:18:41.392 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-06-01T02:18:41.392 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pidfile_write: ignore empty --pid-file 2024-06-01T02:18:41.392 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: load: jerasure load: lrc 2024-06-01T02:18:41.392 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: RocksDB version: 6.15.5 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Compile date Apr 18 2022 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: DB SUMMARY 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: DB Session ID: U5L348GR0UCRZ2Y3KAJB 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: CURRENT file: CURRENT 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: IDENTITY file: IDENTITY 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi027/store.db dir, Total Num: 0, files: 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi027/store.db: 000004.log size: 511 ; 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.error_if_exists: 0 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.create_if_missing: 0 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.paranoid_checks: 1 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.env: 0x564691ec6860 2024-06-01T02:18:41.393 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.fs: Posix File System 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.info_log: 0x564694227fa0 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_file_opening_threads: 16 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.statistics: (nil) 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.use_fsync: 0 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_log_file_size: 0 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.keep_log_file_num: 1000 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.recycle_log_file_num: 0 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.allow_fallocate: 1 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.allow_mmap_reads: 0 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.allow_mmap_writes: 0 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.use_direct_reads: 0 2024-06-01T02:18:41.394 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-01T02:18:41.395 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.create_missing_column_families: 0 2024-06-01T02:18:41.395 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.db_log_dir: 2024-06-01T02:18:41.395 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi027/store.db 2024-06-01T02:18:41.395 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-01T02:18:41.395 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-01T02:18:41.395 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-01T02:18:41.395 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-01T02:18:41.395 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-01T02:18:41.395 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-01T02:18:41.397 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.advise_random_on_open: 1 2024-06-01T02:18:41.397 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.db_write_buffer_size: 0 2024-06-01T02:18:41.397 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.write_buffer_manager: 0x564694318330 2024-06-01T02:18:41.397 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-01T02:18:41.397 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-06-01T02:18:41.397 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-01T02:18:41.397 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-01T02:18:41.397 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.rate_limiter: (nil) 2024-06-01T02:18:41.397 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.wal_recovery_mode: 2 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.enable_thread_tracking: 0 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.enable_pipelined_write: 0 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.unordered_write: 0 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.row_cache: None 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.wal_filter: None 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.allow_ingest_behind: 0 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.preserve_deletes: 0 2024-06-01T02:18:41.398 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.two_write_queues: 0 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.manual_wal_flush: 0 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.atomic_flush: 0 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.log_readahead_size: 0 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.best_efforts_recovery: 0 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.allow_data_in_errors: 0 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.db_host_id: __hostname__ 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_background_jobs: 2 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_background_compactions: -1 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_subcompactions: 1 2024-06-01T02:18:41.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_total_wal_size: 0 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_open_files: -1 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bytes_per_sync: 0 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_readahead_size: 0 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_background_flushes: -1 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Compression algorithms supported: 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-01T02:18:41.400 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: kZSTD supported: 0 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: kXpressCompression supported: 0 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: kLZ4HCCompression supported: 1 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: kLZ4Compression supported: 1 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: kBZip2Compression supported: 0 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: kZlibCompression supported: 1 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: kSnappyCompression supported: 1 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi027/store.db/MANIFEST-000003 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.merge_operator: 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_filter: None 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_filter_factory: None 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.sst_partitioner_factory: None 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-01T02:18:41.401 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5646941f3d00) 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: index_type: 0 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: data_block_index_type: 0 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: index_shortening: 1 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: hash_index_allow_collision: 1 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: checksum: 1 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: no_block_cache: 0 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_cache: 0x56469425e170 2024-06-01T02:18:41.402 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_cache_options: 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: capacity : 536870912 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: num_shard_bits : 4 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: strict_capacity_limit : 0 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_cache_compressed: (nil) 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: persistent_cache: (nil) 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_size: 4096 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_size_deviation: 10 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_restart_interval: 16 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: index_block_restart_interval: 1 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: metadata_block_size: 4096 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: partition_filters: 0 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: use_delta_encoding: 1 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-06-01T02:18:41.403 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: whole_key_filtering: 1 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: verify_compression: 0 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: format_version: 4 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: enable_index_compression: 1 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_align: 0 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.write_buffer_size: 33554432 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_write_buffer_number: 2 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compression: NoCompression 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bottommost_compression: Disabled 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.prefix_extractor: nullptr 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.num_levels: 7 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-01T02:18:41.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compression_opts.level: 32767 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compression_opts.strategy: 0 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compression_opts.enabled: false 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-01T02:18:41.405 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.target_file_size_base: 67108864 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-01T02:18:41.406 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.arena_block_size: 4194304 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.disable_auto_compactions: 0 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.table_properties_collectors: 2024-06-01T02:18:41.407 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.inplace_update_support: 0 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.bloom_locality: 0 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.max_successive_merges: 0 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.paranoid_file_checks: 0 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.force_consistency_checks: 1 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.report_bg_io_stats: 0 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.ttl: 2592000 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.enable_blob_files: false 2024-06-01T02:18:41.408 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.min_blob_size: 0 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.blob_file_size: 268435456 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi027/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717208321183854, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717208321184166, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1717208321, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "19bf49a4-ecec-48b2-80ab-f910aa86e7f1", "db_session_id": "U5L348GR0UCRZ2Y3KAJB"}} 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717208321184473, "job": 1, "event": "recovery_finished"} 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi027/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x564694244a80 2024-06-01T02:18:41.409 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: DB pointer 0x564694254000 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** DB Stats ** 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-01T02:18:41.410 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-01T02:18:41.411 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Cumulative compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Interval compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:18:41.412 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Cumulative compaction: 0.00 GB write, 0.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027 does not exist in monmap, will attempt to join an existing cluster 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: using public_addr v2:172.21.15.27:0/0 -> [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: starting mon.smithi027 rank -1 at public addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] at bind addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi027 fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:18:41.413 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(???) e0 preinit fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).mds e1 new map 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).mds e1 print_map 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: e1 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: legacy client fscid: -1 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: No filesystems configured 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-06-01T02:18:41.414 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Deploying daemon crash.smithi003 on smithi003 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/419914826' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Deploying daemon grafana.smithi003 on smithi003 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/419914826' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: osdmap e4: 0 total, 0 up, 0 in 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi003:172.21.15.3=smithi003;smithi027:172.21.15.27=smithi027", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Saving service mon spec with placement smithi003:172.21.15.3=smithi003;smithi027:172.21.15.27=smithi027;count:2 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/4258937747' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/3909526768' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.416 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/3089567829' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Deploying daemon node-exporter.smithi003 on smithi003 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/3313471893' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14160 172.21.15.3:0/2019140665' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mgrmap e14: smithi003.njsqxz(active, since 47s) 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/2575397509' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/3861241531' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Active manager daemon smithi003.njsqxz restarted 2024-06-01T02:18:41.417 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Activating manager daemon smithi003.njsqxz 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: osdmap e5: 0 total, 0 up, 0 in 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mgrmap e15: smithi003.njsqxz(active, starting, since 0.00964875s) 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Manager daemon smithi003.njsqxz is now available 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/2944014260' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mgrmap e16: smithi003.njsqxz(active, since 1.01267s) 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: [01/Jun/2024:02:18:25] ENGINE Bus STARTING 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: [01/Jun/2024:02:18:26] ENGINE Serving on https://172.21.15.3:7150 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: [01/Jun/2024:02:18:26] ENGINE Bus STARTED 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mgrmap e17: smithi003.njsqxz(active, since 2s) 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd/host:smithi003", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:18:41.419 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Updating smithi003:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd/host:smithi027", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Deploying daemon crash.smithi027 on smithi027 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/1172041388' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:18:41.420 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Deploying daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/4275143857' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: Deploying daemon mon.smithi027 on smithi027 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/2037932566' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 ceph-mon[27271]: mon.smithi027@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 bash[27234]: edc537c54a3ad9f24f56782b13871c4c220ec52f067649c1356d66c541103ede 2024-06-01T02:18:41.421 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:41 smithi027 systemd[1]: Started Ceph mon.smithi027 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:18:42.841 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-01T02:18:42.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-01T02:18:46.524 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.524 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:18:46.524 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: mon.smithi003 calling monitor election 2024-06-01T02:18:46.524 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/1842706958' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: mon.smithi027 calling monitor election 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: mon.smithi003 is new leader, mons smithi003,smithi027 in quorum (ranks 0,1) 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: monmap e2: 2 mons at {smithi003=[v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0],smithi027=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: fsmap 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: osdmap e5: 0 total, 0 up, 0 in 2024-06-01T02:18:46.525 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: mgrmap e17: smithi003.njsqxz(active, since 21s) 2024-06-01T02:18:46.526 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:18:46.526 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/1842706958' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:18:46.526 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: overall HEALTH_OK 2024-06-01T02:18:46.526 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/1842706958' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:18:46.526 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/1842706958' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:18:46.526 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:18:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: mon.smithi003 calling monitor election 2024-06-01T02:18:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/1842706958' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:18:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: mon.smithi027 calling monitor election 2024-06-01T02:18:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: mon.smithi003 is new leader, mons smithi003,smithi027 in quorum (ranks 0,1) 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: monmap e2: 2 mons at {smithi003=[v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0],smithi027=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: fsmap 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: osdmap e5: 0 total, 0 up, 0 in 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: mgrmap e17: smithi003.njsqxz(active, since 21s) 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/1842706958' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: overall HEALTH_OK 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/1842706958' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/1842706958' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:18:46.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:47.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:47 smithi003 ceph-mon[25122]: Deploying daemon node-exporter.smithi027 on smithi027 2024-06-01T02:18:47.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:47 smithi003 ceph-mon[25122]: mgrmap e18: smithi003.njsqxz(active, since 21s), standbys: smithi027.tujskj 2024-06-01T02:18:47.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:47 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi027.tujskj", "id": "smithi027.tujskj"}]: dispatch 2024-06-01T02:18:47.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:47 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:47.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:47 smithi027 ceph-mon[27271]: Deploying daemon node-exporter.smithi027 on smithi027 2024-06-01T02:18:47.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:47 smithi027 ceph-mon[27271]: mgrmap e18: smithi003.njsqxz(active, since 21s), standbys: smithi027.tujskj 2024-06-01T02:18:47.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:47 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi027.tujskj", "id": "smithi027.tujskj"}]: dispatch 2024-06-01T02:18:47.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:47 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:18:47.790 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-06-01T02:18:47.790 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":2,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","modified":"2024-06-01T02:18:41.208406Z","created":"2024-06-01T02:16:35.108780Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi003","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:3300","nonce":0},{"type":"v1","addr":"172.21.15.3:6789","nonce":0}]},"addr":"172.21.15.3:6789/0","public_addr":"172.21.15.3:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi027","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-06-01T02:18:47.791 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 2 2024-06-01T02:18:48.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:48 smithi003 ceph-mon[25122]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:48.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:48 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/1013883272' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:48.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:48 smithi027 ceph-mon[27271]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:48.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:48 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/1013883272' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-01T02:18:49.345 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-06-01T02:18:49.346 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph config generate-minimal-conf 2024-06-01T02:18:49.945 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:49 smithi003 ceph-mon[25122]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:50.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:49 smithi027 ceph-mon[27271]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:51.230 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:50 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:51.230 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:50 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:51.231 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:50 smithi003 ceph-mon[25122]: Deploying daemon prometheus.smithi003 on smithi003 2024-06-01T02:18:51.231 INFO:teuthology.orchestra.run.smithi003.stdout:# minimal ceph.conf for e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:18:51.231 INFO:teuthology.orchestra.run.smithi003.stdout:[global] 2024-06-01T02:18:51.231 INFO:teuthology.orchestra.run.smithi003.stdout: fsid = e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:18:51.231 INFO:teuthology.orchestra.run.smithi003.stdout: mon_host = [v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0] [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] 2024-06-01T02:18:51.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:50 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:51.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:50 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:51.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:50 smithi027 ceph-mon[27271]: Deploying daemon prometheus.smithi003 on smithi003 2024-06-01T02:18:51.796 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-06-01T02:18:51.796 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:18:51.796 DEBUG:teuthology.orchestra.run.smithi003:> sudo dd of=/etc/ceph/ceph.conf 2024-06-01T02:18:51.825 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:18:51.825 DEBUG:teuthology.orchestra.run.smithi003:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:18:51.890 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-01T02:18:51.891 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.conf 2024-06-01T02:18:51.925 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-01T02:18:51.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:18:51.995 INFO:tasks.cephadm:Deploying OSDs... 2024-06-01T02:18:51.995 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:18:51.995 DEBUG:teuthology.orchestra.run.smithi003:> dd if=/scratch_devs of=/dev/stdout 2024-06-01T02:18:52.012 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-06-01T02:18:52.012 DEBUG:teuthology.orchestra.run.smithi003:> stat /dev/vg_nvme/lv_1 2024-06-01T02:18:52.068 INFO:teuthology.orchestra.run.smithi003.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-06-01T02:18:52.068 INFO:teuthology.orchestra.run.smithi003.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-01T02:18:52.068 INFO:teuthology.orchestra.run.smithi003.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-06-01T02:18:52.068 INFO:teuthology.orchestra.run.smithi003.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-01T02:18:52.068 INFO:teuthology.orchestra.run.smithi003.stdout:Context: system_u:object_r:device_t:s0 2024-06-01T02:18:52.068 INFO:teuthology.orchestra.run.smithi003.stdout:Access: 2024-06-01 02:18:15.161356387 +0000 2024-06-01T02:18:52.069 INFO:teuthology.orchestra.run.smithi003.stdout:Modify: 2024-06-01 02:15:19.794144287 +0000 2024-06-01T02:18:52.069 INFO:teuthology.orchestra.run.smithi003.stdout:Change: 2024-06-01 02:15:19.794144287 +0000 2024-06-01T02:18:52.069 INFO:teuthology.orchestra.run.smithi003.stdout: Birth: 2024-06-01 02:15:19.794144287 +0000 2024-06-01T02:18:52.069 DEBUG:teuthology.orchestra.run.smithi003:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-06-01T02:18:52.133 INFO:teuthology.orchestra.run.smithi003.stderr:1+0 records in 2024-06-01T02:18:52.133 INFO:teuthology.orchestra.run.smithi003.stderr:1+0 records out 2024-06-01T02:18:52.133 INFO:teuthology.orchestra.run.smithi003.stderr:512 bytes copied, 0.000257311 s, 2.0 MB/s 2024-06-01T02:18:52.134 DEBUG:teuthology.orchestra.run.smithi003:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-06-01T02:18:52.195 DEBUG:teuthology.orchestra.run.smithi003:> stat /dev/vg_nvme/lv_2 2024-06-01T02:18:52.253 INFO:teuthology.orchestra.run.smithi003.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-06-01T02:18:52.253 INFO:teuthology.orchestra.run.smithi003.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-01T02:18:52.253 INFO:teuthology.orchestra.run.smithi003.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-06-01T02:18:52.253 INFO:teuthology.orchestra.run.smithi003.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-01T02:18:52.253 INFO:teuthology.orchestra.run.smithi003.stdout:Context: system_u:object_r:device_t:s0 2024-06-01T02:18:52.253 INFO:teuthology.orchestra.run.smithi003.stdout:Access: 2024-06-01 02:18:15.161356387 +0000 2024-06-01T02:18:52.253 INFO:teuthology.orchestra.run.smithi003.stdout:Modify: 2024-06-01 02:15:20.143139815 +0000 2024-06-01T02:18:52.253 INFO:teuthology.orchestra.run.smithi003.stdout:Change: 2024-06-01 02:15:20.143139815 +0000 2024-06-01T02:18:52.253 INFO:teuthology.orchestra.run.smithi003.stdout: Birth: 2024-06-01 02:15:20.143139815 +0000 2024-06-01T02:18:52.253 DEBUG:teuthology.orchestra.run.smithi003:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-06-01T02:18:52.321 INFO:teuthology.orchestra.run.smithi003.stderr:1+0 records in 2024-06-01T02:18:52.321 INFO:teuthology.orchestra.run.smithi003.stderr:1+0 records out 2024-06-01T02:18:52.321 INFO:teuthology.orchestra.run.smithi003.stderr:512 bytes copied, 0.00021077 s, 2.4 MB/s 2024-06-01T02:18:52.323 DEBUG:teuthology.orchestra.run.smithi003:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-06-01T02:18:52.381 DEBUG:teuthology.orchestra.run.smithi003:> stat /dev/vg_nvme/lv_3 2024-06-01T02:18:52.438 INFO:teuthology.orchestra.run.smithi003.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-06-01T02:18:52.438 INFO:teuthology.orchestra.run.smithi003.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-01T02:18:52.438 INFO:teuthology.orchestra.run.smithi003.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-06-01T02:18:52.438 INFO:teuthology.orchestra.run.smithi003.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-01T02:18:52.438 INFO:teuthology.orchestra.run.smithi003.stdout:Context: system_u:object_r:device_t:s0 2024-06-01T02:18:52.438 INFO:teuthology.orchestra.run.smithi003.stdout:Access: 2024-06-01 02:18:15.162356374 +0000 2024-06-01T02:18:52.438 INFO:teuthology.orchestra.run.smithi003.stdout:Modify: 2024-06-01 02:15:20.482135471 +0000 2024-06-01T02:18:52.438 INFO:teuthology.orchestra.run.smithi003.stdout:Change: 2024-06-01 02:15:20.482135471 +0000 2024-06-01T02:18:52.438 INFO:teuthology.orchestra.run.smithi003.stdout: Birth: 2024-06-01 02:15:20.482135471 +0000 2024-06-01T02:18:52.438 DEBUG:teuthology.orchestra.run.smithi003:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-06-01T02:18:52.502 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:52 smithi003 ceph-mon[25122]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:52.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:52 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/10048414' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:52.505 INFO:teuthology.orchestra.run.smithi003.stderr:1+0 records in 2024-06-01T02:18:52.506 INFO:teuthology.orchestra.run.smithi003.stderr:1+0 records out 2024-06-01T02:18:52.506 INFO:teuthology.orchestra.run.smithi003.stderr:512 bytes copied, 0.000172499 s, 3.0 MB/s 2024-06-01T02:18:52.507 DEBUG:teuthology.orchestra.run.smithi003:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-06-01T02:18:52.565 DEBUG:teuthology.orchestra.run.smithi003:> stat /dev/vg_nvme/lv_4 2024-06-01T02:18:52.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:52 smithi027 ceph-mon[27271]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:52.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:52 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/10048414' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:18:52.622 INFO:teuthology.orchestra.run.smithi003.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-06-01T02:18:52.622 INFO:teuthology.orchestra.run.smithi003.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-01T02:18:52.622 INFO:teuthology.orchestra.run.smithi003.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-06-01T02:18:52.622 INFO:teuthology.orchestra.run.smithi003.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-01T02:18:52.622 INFO:teuthology.orchestra.run.smithi003.stdout:Context: system_u:object_r:device_t:s0 2024-06-01T02:18:52.623 INFO:teuthology.orchestra.run.smithi003.stdout:Access: 2024-06-01 02:18:15.162356374 +0000 2024-06-01T02:18:52.623 INFO:teuthology.orchestra.run.smithi003.stdout:Modify: 2024-06-01 02:15:20.818131166 +0000 2024-06-01T02:18:52.623 INFO:teuthology.orchestra.run.smithi003.stdout:Change: 2024-06-01 02:15:20.818131166 +0000 2024-06-01T02:18:52.623 INFO:teuthology.orchestra.run.smithi003.stdout: Birth: 2024-06-01 02:15:20.818131166 +0000 2024-06-01T02:18:52.623 DEBUG:teuthology.orchestra.run.smithi003:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-06-01T02:18:52.691 INFO:teuthology.orchestra.run.smithi003.stderr:1+0 records in 2024-06-01T02:18:52.691 INFO:teuthology.orchestra.run.smithi003.stderr:1+0 records out 2024-06-01T02:18:52.691 INFO:teuthology.orchestra.run.smithi003.stderr:512 bytes copied, 0.000180092 s, 2.8 MB/s 2024-06-01T02:18:52.692 DEBUG:teuthology.orchestra.run.smithi003:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-06-01T02:18:52.750 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-06-01T02:18:52.751 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/scratch_devs of=/dev/stdout 2024-06-01T02:18:52.771 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-06-01T02:18:52.771 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_1 2024-06-01T02:18:52.830 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-06-01T02:18:52.830 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-01T02:18:52.830 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-06-01T02:18:52.830 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-01T02:18:52.830 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-06-01T02:18:52.830 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-06-01 02:18:46.813486787 +0000 2024-06-01T02:18:52.830 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-06-01 02:15:16.760936152 +0000 2024-06-01T02:18:52.830 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-06-01 02:15:16.760936152 +0000 2024-06-01T02:18:52.830 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-06-01 02:15:16.760936152 +0000 2024-06-01T02:18:52.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-06-01T02:18:52.901 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-06-01T02:18:52.901 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-06-01T02:18:52.901 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000232555 s, 2.2 MB/s 2024-06-01T02:18:52.902 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-06-01T02:18:52.960 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_2 2024-06-01T02:18:53.017 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-06-01T02:18:53.017 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-01T02:18:53.017 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-06-01T02:18:53.017 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-01T02:18:53.017 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-06-01T02:18:53.018 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-06-01 02:18:46.814486760 +0000 2024-06-01T02:18:53.018 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-06-01 02:15:17.111926633 +0000 2024-06-01T02:18:53.018 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-06-01 02:15:17.111926633 +0000 2024-06-01T02:18:53.018 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-06-01 02:15:17.111926633 +0000 2024-06-01T02:18:53.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-06-01T02:18:53.082 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-06-01T02:18:53.082 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-06-01T02:18:53.082 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000197558 s, 2.6 MB/s 2024-06-01T02:18:53.084 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-06-01T02:18:53.141 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_3 2024-06-01T02:18:53.198 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-06-01T02:18:53.198 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-01T02:18:53.198 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-06-01T02:18:53.198 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-01T02:18:53.198 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-06-01T02:18:53.198 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-06-01 02:18:46.814486760 +0000 2024-06-01T02:18:53.198 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-06-01 02:15:17.451917413 +0000 2024-06-01T02:18:53.198 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-06-01 02:15:17.451917413 +0000 2024-06-01T02:18:53.198 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-06-01 02:15:17.451917413 +0000 2024-06-01T02:18:53.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-06-01T02:18:53.262 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-06-01T02:18:53.262 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-06-01T02:18:53.262 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000192325 s, 2.7 MB/s 2024-06-01T02:18:53.263 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-06-01T02:18:53.320 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_4 2024-06-01T02:18:53.376 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-06-01T02:18:53.377 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-01T02:18:53.377 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-06-01T02:18:53.377 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-01T02:18:53.377 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-06-01T02:18:53.377 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-06-01 02:18:46.814486760 +0000 2024-06-01T02:18:53.377 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-06-01 02:15:17.793908138 +0000 2024-06-01T02:18:53.377 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-06-01 02:15:17.793908138 +0000 2024-06-01T02:18:53.377 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-06-01 02:15:17.793908138 +0000 2024-06-01T02:18:53.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-06-01T02:18:53.440 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-06-01T02:18:53.441 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-06-01T02:18:53.441 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.00019378 s, 2.6 MB/s 2024-06-01T02:18:53.442 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-06-01T02:18:53.500 INFO:tasks.cephadm:Deploying osd.0 on smithi003 with /dev/vg_nvme/lv_4... 2024-06-01T02:18:53.500 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-06-01T02:18:54.553 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:54 smithi003 ceph-mon[25122]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:54.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:54 smithi027 ceph-mon[27271]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:56.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:55 smithi003 ceph-mon[25122]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:56.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:56.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:55 smithi027 ceph-mon[27271]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:56.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:18:57.388 INFO:teuthology.orchestra.run.smithi003.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-06-01T02:18:57.388 INFO:teuthology.orchestra.run.smithi003.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-06-01T02:18:57.388 INFO:teuthology.orchestra.run.smithi003.stderr: stderr: 10+0 records in 2024-06-01T02:18:57.388 INFO:teuthology.orchestra.run.smithi003.stderr:10+0 records out 2024-06-01T02:18:57.388 INFO:teuthology.orchestra.run.smithi003.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112379 s, 933 MB/s 2024-06-01T02:18:57.388 INFO:teuthology.orchestra.run.smithi003.stderr:--> Zapping successful for: 2024-06-01T02:18:58.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:57 smithi003 ceph-mon[25122]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:18:58.288 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi003:vg_nvme/lv_4 2024-06-01T02:18:58.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:57 smithi027 ceph-mon[27271]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:00.072 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:18:59 smithi003 ceph-mon[25122]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:00.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:18:59 smithi027 ceph-mon[27271]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:02.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:01 smithi003 ceph-mon[25122]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:02.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:01 smithi027 ceph-mon[27271]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:03.120 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:02 smithi003 ceph-mon[25122]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi003:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:19:03.120 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:19:03.120 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:19:03.120 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:03.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:02 smithi027 ceph-mon[27271]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi003:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:19:03.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:19:03.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:19:03.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:03.899 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:03 smithi003 ceph-mon[25122]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:03.899 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:03 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:03.899 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:03 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:19:03.899 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:03 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:03.899 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:03 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:19:04.210 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:03 smithi027 ceph-mon[27271]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:04.210 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:03 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:04.210 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:03 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:19:04.210 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:03 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:04.210 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:03 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:19:05.758 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:05.758 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:05 smithi003 ceph-mon[25122]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-01T02:19:05.758 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:05.758 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:05.758 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:05 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3161926111' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd50417e-a988-4f23-87d1-e45084b46b59"}]: dispatch 2024-06-01T02:19:05.759 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:05 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3161926111' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fd50417e-a988-4f23-87d1-e45084b46b59"}]': finished 2024-06-01T02:19:05.759 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:05 smithi003 ceph-mon[25122]: osdmap e6: 1 total, 0 up, 1 in 2024-06-01T02:19:05.759 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:05.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:05.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:05 smithi027 ceph-mon[27271]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-01T02:19:05.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:05.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:05.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:05 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/3161926111' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fd50417e-a988-4f23-87d1-e45084b46b59"}]: dispatch 2024-06-01T02:19:05.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:05 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/3161926111' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fd50417e-a988-4f23-87d1-e45084b46b59"}]': finished 2024-06-01T02:19:05.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:05 smithi027 ceph-mon[27271]: osdmap e6: 1 total, 0 up, 1 in 2024-06-01T02:19:05.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:06.660 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:06 smithi003 ceph-mon[25122]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:19:06.661 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:06 smithi003 ceph-mon[25122]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:06.661 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:06 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3997154888' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:19:06.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:06 smithi027 ceph-mon[27271]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:19:06.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:06 smithi027 ceph-mon[27271]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:06.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:06 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/3997154888' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:19:08.737 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:08 smithi003 ceph-mon[25122]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:08.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:08 smithi027 ceph-mon[27271]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:09.975 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:09 smithi003 ceph-mon[25122]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:10.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:09 smithi027 ceph-mon[27271]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:12.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:12.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:12.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:19:12.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:12.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:12.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: Reconfiguring mon.smithi003 (unknown last config time)... 2024-06-01T02:19:12.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:19:12.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:19:12.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:12.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:11 smithi003 ceph-mon[25122]: Reconfiguring daemon mon.smithi003 on smithi003 2024-06-01T02:19:12.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:12.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:12.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:19:12.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:12.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:12.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: Reconfiguring mon.smithi003 (unknown last config time)... 2024-06-01T02:19:12.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:19:12.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:19:12.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:12.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:11 smithi027 ceph-mon[27271]: Reconfiguring daemon mon.smithi003 on smithi003 2024-06-01T02:19:13.080 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:12 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-01T02:19:13.080 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:12 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:13.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:12 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-01T02:19:13.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:12 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:14.229 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:13 smithi003 ceph-mon[25122]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:14.229 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:13 smithi003 ceph-mon[25122]: Deploying daemon osd.0 on smithi003 2024-06-01T02:19:14.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:13 smithi027 ceph-mon[27271]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:14.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:13 smithi027 ceph-mon[27271]: Deploying daemon osd.0 on smithi003 2024-06-01T02:19:15.314 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:15.314 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:15 smithi003 ceph-mon[25122]: Reconfiguring mgr.smithi003.njsqxz (unknown last config time)... 2024-06-01T02:19:15.314 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi003.njsqxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:19:15.314 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:19:15.314 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:15.315 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:15 smithi003 ceph-mon[25122]: Reconfiguring daemon mgr.smithi003.njsqxz on smithi003 2024-06-01T02:19:15.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:15.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:15 smithi027 ceph-mon[27271]: Reconfiguring mgr.smithi003.njsqxz (unknown last config time)... 2024-06-01T02:19:15.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi003.njsqxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:19:15.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:19:15.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:15.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:15 smithi027 ceph-mon[27271]: Reconfiguring daemon mgr.smithi003.njsqxz on smithi003 2024-06-01T02:19:16.141 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:16 smithi003 ceph-mon[25122]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:16.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:16 smithi027 ceph-mon[27271]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:18.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:18 smithi003 ceph-mon[25122]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:18 smithi027 ceph-mon[27271]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:20.136 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:19 smithi003 ceph-mon[25122]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:20.136 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:19 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:20.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:20 smithi027 ceph-mon[27271]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:20.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:20 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:21.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:21 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:21.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:21 smithi003 ceph-mon[25122]: Reconfiguring alertmanager.smithi003 (dependencies changed)... 2024-06-01T02:19:21.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:21 smithi003 ceph-mon[25122]: Reconfiguring daemon alertmanager.smithi003 on smithi003 2024-06-01T02:19:21.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:21 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:21.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:21 smithi027 ceph-mon[27271]: Reconfiguring alertmanager.smithi003 (dependencies changed)... 2024-06-01T02:19:21.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:21 smithi027 ceph-mon[27271]: Reconfiguring daemon alertmanager.smithi003 on smithi003 2024-06-01T02:19:22.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:22 smithi003 ceph-mon[25122]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:22.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:22 smithi003 ceph-mon[25122]: from='osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-01T02:19:22.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:22 smithi027 ceph-mon[27271]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:22.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:22 smithi027 ceph-mon[27271]: from='osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-01T02:19:23.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:23 smithi003 ceph-mon[25122]: from='osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-01T02:19:23.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:23 smithi003 ceph-mon[25122]: osdmap e7: 1 total, 0 up, 1 in 2024-06-01T02:19:23.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:23 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:23.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:23 smithi003 ceph-mon[25122]: from='osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:19:23.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:23 smithi027 ceph-mon[27271]: from='osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-01T02:19:23.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:23 smithi027 ceph-mon[27271]: osdmap e7: 1 total, 0 up, 1 in 2024-06-01T02:19:23.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:23 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:23.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:23 smithi027 ceph-mon[27271]: from='osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:19:24.328 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:24 smithi003 ceph-mon[25122]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:24.329 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:24 smithi003 ceph-mon[25122]: from='osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]': finished 2024-06-01T02:19:24.329 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:24 smithi003 ceph-mon[25122]: osdmap e8: 1 total, 0 up, 1 in 2024-06-01T02:19:24.329 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:24.329 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:24.329 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:24.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:24 smithi027 ceph-mon[27271]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:24.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:24 smithi027 ceph-mon[27271]: from='osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]': finished 2024-06-01T02:19:24.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:24 smithi027 ceph-mon[27271]: osdmap e8: 1 total, 0 up, 1 in 2024-06-01T02:19:24.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:24.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:24.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:25.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:25 smithi003 ceph-mon[25122]: purged_snaps scrub starts 2024-06-01T02:19:25.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:25 smithi003 ceph-mon[25122]: purged_snaps scrub ok 2024-06-01T02:19:25.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:25 smithi003 ceph-mon[25122]: osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731] boot 2024-06-01T02:19:25.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:25 smithi003 ceph-mon[25122]: osdmap e9: 1 total, 1 up, 1 in 2024-06-01T02:19:25.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:25.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:19:25.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:19:25.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:25 smithi027 ceph-mon[27271]: purged_snaps scrub starts 2024-06-01T02:19:25.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:25 smithi027 ceph-mon[27271]: purged_snaps scrub ok 2024-06-01T02:19:25.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:25 smithi027 ceph-mon[27271]: osd.0 [v2:172.21.15.3:6802/2738410731,v1:172.21.15.3:6803/2738410731] boot 2024-06-01T02:19:25.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:25 smithi027 ceph-mon[27271]: osdmap e9: 1 total, 1 up, 1 in 2024-06-01T02:19:25.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:19:25.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:19:25.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:19:26.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:26 smithi003 ceph-mon[25122]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:26.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:26 smithi003 ceph-mon[25122]: osdmap e10: 1 total, 1 up, 1 in 2024-06-01T02:19:26.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:26.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi003", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:19:26.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:26.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:26 smithi027 ceph-mon[27271]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-01T02:19:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:26 smithi027 ceph-mon[27271]: osdmap e10: 1 total, 1 up, 1 in 2024-06-01T02:19:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi003", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:19:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:27.035 INFO:teuthology.orchestra.run.smithi003.stdout:Created osd(s) 0 on host 'smithi003' 2024-06-01T02:19:27.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:27 smithi003 ceph-mon[25122]: Reconfiguring crash.smithi003 (monmap changed)... 2024-06-01T02:19:27.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:27 smithi003 ceph-mon[25122]: Reconfiguring daemon crash.smithi003 on smithi003 2024-06-01T02:19:27.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:27 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:27.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:27 smithi027 ceph-mon[27271]: Reconfiguring crash.smithi003 (monmap changed)... 2024-06-01T02:19:27.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:27 smithi027 ceph-mon[27271]: Reconfiguring daemon crash.smithi003 on smithi003 2024-06-01T02:19:27.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:27 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:27.837 DEBUG:teuthology.orchestra.run.smithi003:osd.0> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.0.service 2024-06-01T02:19:27.840 INFO:tasks.cephadm:Deploying osd.1 on smithi003 with /dev/vg_nvme/lv_3... 2024-06-01T02:19:27.840 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-06-01T02:19:28.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:28 smithi003 ceph-mon[25122]: pgmap v29: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:28.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:28 smithi027 ceph-mon[27271]: pgmap v29: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:30.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:29 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:30.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:29 smithi003 ceph-mon[25122]: Reconfiguring grafana.smithi003 (dependencies changed)... 2024-06-01T02:19:30.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:29 smithi003 ceph-mon[25122]: Reconfiguring daemon grafana.smithi003 on smithi003 2024-06-01T02:19:30.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:29 smithi003 ceph-mon[25122]: pgmap v30: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:30.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:29 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:30.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:29 smithi027 ceph-mon[27271]: Reconfiguring grafana.smithi003 (dependencies changed)... 2024-06-01T02:19:30.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:29 smithi027 ceph-mon[27271]: Reconfiguring daemon grafana.smithi003 on smithi003 2024-06-01T02:19:30.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:29 smithi027 ceph-mon[27271]: pgmap v30: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:31.207 INFO:teuthology.orchestra.run.smithi003.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-06-01T02:19:31.207 INFO:teuthology.orchestra.run.smithi003.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-06-01T02:19:31.207 INFO:teuthology.orchestra.run.smithi003.stderr: stderr: 10+0 records in 2024-06-01T02:19:31.207 INFO:teuthology.orchestra.run.smithi003.stderr:10+0 records out 2024-06-01T02:19:31.208 INFO:teuthology.orchestra.run.smithi003.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0127636 s, 822 MB/s 2024-06-01T02:19:31.208 INFO:teuthology.orchestra.run.smithi003.stderr:--> Zapping successful for: 2024-06-01T02:19:32.027 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi003:vg_nvme/lv_3 2024-06-01T02:19:32.148 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:31 smithi003 ceph-mon[25122]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:32.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:31 smithi027 ceph-mon[27271]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:34.006 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:33 smithi003 ceph-mon[25122]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:34.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:33 smithi027 ceph-mon[27271]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:36.061 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:35 smithi003 ceph-mon[25122]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:36.062 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:35 smithi003 ceph-mon[25122]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi003:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:19:36.062 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:35 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:19:36.062 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:35 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:19:36.062 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:35 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:36.062 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:35 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:36.062 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:35 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:19:36.062 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:35 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:36.181 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:35 smithi027 ceph-mon[27271]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:36.181 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:35 smithi027 ceph-mon[27271]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi003:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:19:36.182 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:35 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:19:36.182 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:35 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:19:36.182 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:35 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:36.182 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:35 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:36.182 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:35 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:19:36.182 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:35 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:36.975 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:36 smithi003 ceph-mon[25122]: Reconfiguring crash.smithi027 (monmap changed)... 2024-06-01T02:19:36.976 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:36 smithi003 ceph-mon[25122]: Reconfiguring daemon crash.smithi027 on smithi027 2024-06-01T02:19:37.281 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:36 smithi027 ceph-mon[27271]: Reconfiguring crash.smithi027 (monmap changed)... 2024-06-01T02:19:37.281 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:36 smithi027 ceph-mon[27271]: Reconfiguring daemon crash.smithi027 on smithi027 2024-06-01T02:19:38.178 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:38.178 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:38.178 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: Reconfiguring mgr.smithi027.tujskj (monmap changed)... 2024-06-01T02:19:38.178 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:19:38.178 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:19:38.178 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:38.178 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: Reconfiguring daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:19:38.178 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1066810112' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "abe11909-0ec2-46d5-9dcb-2234cfbf5e1c"}]: dispatch 2024-06-01T02:19:38.178 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1066810112' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "abe11909-0ec2-46d5-9dcb-2234cfbf5e1c"}]': finished 2024-06-01T02:19:38.179 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: osdmap e11: 2 total, 1 up, 2 in 2024-06-01T02:19:38.179 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:37 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:38.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: Reconfiguring mgr.smithi027.tujskj (monmap changed)... 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: Reconfiguring daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1066810112' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "abe11909-0ec2-46d5-9dcb-2234cfbf5e1c"}]: dispatch 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1066810112' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "abe11909-0ec2-46d5-9dcb-2234cfbf5e1c"}]': finished 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: osdmap e11: 2 total, 1 up, 2 in 2024-06-01T02:19:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:37 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:39.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:39.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:39 smithi003 ceph-mon[25122]: Reconfiguring mon.smithi027 (monmap changed)... 2024-06-01T02:19:39.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:19:39.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:19:39.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:39.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:39 smithi003 ceph-mon[25122]: Reconfiguring daemon mon.smithi027 on smithi027 2024-06-01T02:19:39.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:39 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2749558639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:19:39.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:39.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:39 smithi027 ceph-mon[27271]: Reconfiguring mon.smithi027 (monmap changed)... 2024-06-01T02:19:39.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:19:39.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:19:39.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:39.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:39 smithi027 ceph-mon[27271]: Reconfiguring daemon mon.smithi027 on smithi027 2024-06-01T02:19:39.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:39 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/2749558639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:19:40.641 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:40 smithi003 ceph-mon[25122]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:40.641 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:40 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:40.641 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:40 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:19:40.641 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:40 smithi003 ceph-mon[25122]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:19:40.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:40 smithi027 ceph-mon[27271]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:40.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:40 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:40.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:40 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:19:40.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:40 smithi027 ceph-mon[27271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:19:42.572 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:42 smithi003 ceph-mon[25122]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:42.649 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:42 smithi027 ceph-mon[27271]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:44.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:44 smithi003 ceph-mon[25122]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:44.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:44 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-01T02:19:44.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:44 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:44.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:44 smithi027 ceph-mon[27271]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:44.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:44 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-01T02:19:44.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:44 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:45.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:45 smithi003 ceph-mon[25122]: Deploying daemon osd.1 on smithi003 2024-06-01T02:19:45.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:45 smithi027 ceph-mon[27271]: Deploying daemon osd.1 on smithi003 2024-06-01T02:19:46.743 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:46 smithi003 ceph-mon[25122]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:46.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:46 smithi027 ceph-mon[27271]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:48.740 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:48 smithi003 ceph-mon[25122]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:48.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:48 smithi027 ceph-mon[27271]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:50.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:49 smithi003 ceph-mon[25122]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:50.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:49 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:50.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:49 smithi027 ceph-mon[27271]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:50.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:49 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:51.757 INFO:teuthology.orchestra.run.smithi003.stdout:Created osd(s) 1 on host 'smithi003' 2024-06-01T02:19:52.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:52.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:52.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.3:9093"}]: dispatch 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.3:3000"}]: dispatch 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.3:9095"}]: dispatch 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:52.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:19:52.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.3:9093"}]: dispatch 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.3:3000"}]: dispatch 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.3:9095"}]: dispatch 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:19:52.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:19:52.435 DEBUG:teuthology.orchestra.run.smithi003:osd.1> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.1.service 2024-06-01T02:19:52.438 INFO:tasks.cephadm:Deploying osd.2 on smithi003 with /dev/vg_nvme/lv_2... 2024-06-01T02:19:52.438 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-06-01T02:19:53.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:53 smithi003 ceph-mon[25122]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.3:9093"}]: dispatch 2024-06-01T02:19:53.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:53 smithi003 ceph-mon[25122]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:19:53.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:53 smithi003 ceph-mon[25122]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.3:3000"}]: dispatch 2024-06-01T02:19:53.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:53 smithi003 ceph-mon[25122]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:19:53.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:53 smithi003 ceph-mon[25122]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.3:9095"}]: dispatch 2024-06-01T02:19:53.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:53 smithi003 ceph-mon[25122]: from='osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-01T02:19:53.310 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:53 smithi027 ceph-mon[27271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.3:9093"}]: dispatch 2024-06-01T02:19:53.310 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:53 smithi027 ceph-mon[27271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:19:53.310 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:53 smithi027 ceph-mon[27271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.3:3000"}]: dispatch 2024-06-01T02:19:53.310 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:53 smithi027 ceph-mon[27271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:19:53.310 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:53 smithi027 ceph-mon[27271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.3:9095"}]: dispatch 2024-06-01T02:19:53.311 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:53 smithi027 ceph-mon[27271]: from='osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-01T02:19:54.305 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:54 smithi003 ceph-mon[25122]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:54.305 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:54 smithi003 ceph-mon[25122]: from='osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-01T02:19:54.305 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:54 smithi003 ceph-mon[25122]: osdmap e12: 2 total, 1 up, 2 in 2024-06-01T02:19:54.305 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:54 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:54.305 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:54 smithi003 ceph-mon[25122]: from='osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:19:54.305 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:54 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:54.306 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:19:54 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:19:54.014+0000 7f0a7cb8e700 -1 osd.1 0 waiting for initial osdmap 2024-06-01T02:19:54.306 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:19:54 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:19:54.017+0000 7f0a75521700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:19:54.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:54 smithi027 ceph-mon[27271]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:54.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:54 smithi027 ceph-mon[27271]: from='osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-01T02:19:54.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:54 smithi027 ceph-mon[27271]: osdmap e12: 2 total, 1 up, 2 in 2024-06-01T02:19:54.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:54 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:54.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:54 smithi027 ceph-mon[27271]: from='osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:19:54.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:54 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:19:55.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:55 smithi003 ceph-mon[25122]: from='osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]': finished 2024-06-01T02:19:55.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:55 smithi003 ceph-mon[25122]: osdmap e13: 2 total, 1 up, 2 in 2024-06-01T02:19:55.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:55.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:55.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:55 smithi027 ceph-mon[27271]: from='osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]': finished 2024-06-01T02:19:55.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:55 smithi027 ceph-mon[27271]: osdmap e13: 2 total, 1 up, 2 in 2024-06-01T02:19:55.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:55.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:55.880 INFO:teuthology.orchestra.run.smithi003.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-06-01T02:19:55.880 INFO:teuthology.orchestra.run.smithi003.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-06-01T02:19:55.880 INFO:teuthology.orchestra.run.smithi003.stderr: stderr: 10+0 records in 2024-06-01T02:19:55.880 INFO:teuthology.orchestra.run.smithi003.stderr:10+0 records out 2024-06-01T02:19:55.880 INFO:teuthology.orchestra.run.smithi003.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0119547 s, 877 MB/s 2024-06-01T02:19:55.880 INFO:teuthology.orchestra.run.smithi003.stderr:--> Zapping successful for: 2024-06-01T02:19:56.225 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:56 smithi003 ceph-mon[25122]: purged_snaps scrub starts 2024-06-01T02:19:56.226 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:56 smithi003 ceph-mon[25122]: purged_snaps scrub ok 2024-06-01T02:19:56.226 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:56 smithi003 ceph-mon[25122]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:56.226 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:56 smithi003 ceph-mon[25122]: osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156] boot 2024-06-01T02:19:56.226 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:56 smithi003 ceph-mon[25122]: osdmap e14: 2 total, 2 up, 2 in 2024-06-01T02:19:56.226 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:56 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:56.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:56 smithi027 ceph-mon[27271]: purged_snaps scrub starts 2024-06-01T02:19:56.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:56 smithi027 ceph-mon[27271]: purged_snaps scrub ok 2024-06-01T02:19:56.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:56 smithi027 ceph-mon[27271]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-01T02:19:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:56 smithi027 ceph-mon[27271]: osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156] boot 2024-06-01T02:19:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:56 smithi027 ceph-mon[27271]: osdmap e14: 2 total, 2 up, 2 in 2024-06-01T02:19:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:56 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:19:56.576 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi003:vg_nvme/lv_2 2024-06-01T02:19:57.330 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:57 smithi003 ceph-mon[25122]: osdmap e15: 2 total, 2 up, 2 in 2024-06-01T02:19:57.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:57 smithi027 ceph-mon[27271]: osdmap e15: 2 total, 2 up, 2 in 2024-06-01T02:19:58.293 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:58 smithi003 ceph-mon[25122]: pgmap v49: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:19:58.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:58 smithi027 ceph-mon[27271]: pgmap v49: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:00.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:59 smithi003 ceph-mon[25122]: pgmap v50: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:00.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:59 smithi003 ceph-mon[25122]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi003:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:20:00.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:20:00.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:20:00.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:19:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:00.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:59 smithi027 ceph-mon[27271]: pgmap v50: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:00.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:59 smithi027 ceph-mon[27271]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi003:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:20:00.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:20:00.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:20:00.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:19:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:01.004 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:00 smithi003 ceph-mon[25122]: overall HEALTH_OK 2024-06-01T02:20:01.004 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:00 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:01.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:00 smithi027 ceph-mon[27271]: overall HEALTH_OK 2024-06-01T02:20:01.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:00 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:02.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:02 smithi003 ceph-mon[25122]: pgmap v51: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:02.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:02 smithi027 ceph-mon[27271]: pgmap v51: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:03.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:03 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1763902820' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02170894-eb8b-446e-b3be-72db8229c089"}]: dispatch 2024-06-01T02:20:03.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:03 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1763902820' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02170894-eb8b-446e-b3be-72db8229c089"}]': finished 2024-06-01T02:20:03.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:03 smithi003 ceph-mon[25122]: osdmap e16: 3 total, 2 up, 3 in 2024-06-01T02:20:03.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:03 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:03.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:03 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1763902820' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02170894-eb8b-446e-b3be-72db8229c089"}]: dispatch 2024-06-01T02:20:03.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:03 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1763902820' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02170894-eb8b-446e-b3be-72db8229c089"}]': finished 2024-06-01T02:20:03.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:03 smithi027 ceph-mon[27271]: osdmap e16: 3 total, 2 up, 3 in 2024-06-01T02:20:03.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:03 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:04.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:04 smithi003 ceph-mon[25122]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:04.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:04 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3499551339' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:20:04.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:04 smithi027 ceph-mon[27271]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:04.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:04 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/3499551339' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:20:06.329 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:06 smithi003 ceph-mon[25122]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:06.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:06 smithi027 ceph-mon[27271]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:08.325 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:08 smithi027 ceph-mon[27271]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:08.326 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:08.326 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:08.326 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:08.326 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:08.326 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:08.326 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:08.326 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:08.326 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:08 smithi003 ceph-mon[25122]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:08.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:08.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:09.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:09 smithi003 ceph-mon[25122]: Detected new or changed devices on smithi003 2024-06-01T02:20:09.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:09 smithi003 ceph-mon[25122]: Adjusting osd_memory_target on smithi003 to 6434M 2024-06-01T02:20:09.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:09 smithi027 ceph-mon[27271]: Detected new or changed devices on smithi003 2024-06-01T02:20:09.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:09 smithi027 ceph-mon[27271]: Adjusting osd_memory_target on smithi003 to 6434M 2024-06-01T02:20:10.687 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:10 smithi003 ceph-mon[25122]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:10.688 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:10.688 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-01T02:20:10.688 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:10.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:10 smithi027 ceph-mon[27271]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:10.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:10.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-01T02:20:10.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:11.619 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:11 smithi003 ceph-mon[25122]: Deploying daemon osd.2 on smithi003 2024-06-01T02:20:11.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:11 smithi027 ceph-mon[27271]: Deploying daemon osd.2 on smithi003 2024-06-01T02:20:12.727 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:12 smithi003 ceph-mon[25122]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:12.748 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:12 smithi027 ceph-mon[27271]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:14.569 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:14 smithi003 ceph-mon[25122]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:14.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:14 smithi027 ceph-mon[27271]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:16.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:15 smithi003 ceph-mon[25122]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:16.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:16.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:15 smithi027 ceph-mon[27271]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:16.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:18.115 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:17 smithi003 ceph-mon[25122]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:18.116 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:17 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:17 smithi027 ceph-mon[27271]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:17 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:20.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:20 smithi003 ceph-mon[25122]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:20.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:20 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:20.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:20 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:20.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:20 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:20.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:20 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:20.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:20 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:20.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:20 smithi027 ceph-mon[27271]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:20.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:20 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:20.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:20 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:20 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:20 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:20 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:21.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:21 smithi027 ceph-mon[27271]: from='osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-01T02:20:21.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:21 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:21.656 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:21 smithi003 ceph-mon[25122]: from='osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-01T02:20:21.657 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:21 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:21.691 INFO:teuthology.orchestra.run.smithi003.stdout:Created osd(s) 2 on host 'smithi003' 2024-06-01T02:20:22.341 DEBUG:teuthology.orchestra.run.smithi003:osd.2> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.2.service 2024-06-01T02:20:22.343 INFO:tasks.cephadm:Deploying osd.3 on smithi003 with /dev/vg_nvme/lv_1... 2024-06-01T02:20:22.344 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-06-01T02:20:22.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:22 smithi003 ceph-mon[25122]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:22.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:22 smithi003 ceph-mon[25122]: from='osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-01T02:20:22.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:22 smithi003 ceph-mon[25122]: osdmap e17: 3 total, 2 up, 3 in 2024-06-01T02:20:22.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:22 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:22.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:22 smithi003 ceph-mon[25122]: from='osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:20:22.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:22 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:22.585 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:20:22 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[42096]: 2024-06-01T02:20:22.477+0000 7ff4dac1f700 -1 osd.2 0 waiting for initial osdmap 2024-06-01T02:20:22.585 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:20:22 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[42096]: 2024-06-01T02:20:22.481+0000 7ff4d4db5700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:20:22.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:22 smithi027 ceph-mon[27271]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:22.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:22 smithi027 ceph-mon[27271]: from='osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-01T02:20:22.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:22 smithi027 ceph-mon[27271]: osdmap e17: 3 total, 2 up, 3 in 2024-06-01T02:20:22.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:22 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:22.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:22 smithi027 ceph-mon[27271]: from='osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:20:22.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:22 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:23.729 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:23 smithi003 ceph-mon[25122]: from='osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]': finished 2024-06-01T02:20:23.729 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:23 smithi003 ceph-mon[25122]: osdmap e18: 3 total, 2 up, 3 in 2024-06-01T02:20:23.729 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:23 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:23.729 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:23 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:23.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:23 smithi027 ceph-mon[27271]: from='osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]': finished 2024-06-01T02:20:23.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:23 smithi027 ceph-mon[27271]: osdmap e18: 3 total, 2 up, 3 in 2024-06-01T02:20:23.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:23 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:23.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:23 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:24.794 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:24 smithi003 ceph-mon[25122]: purged_snaps scrub starts 2024-06-01T02:20:24.795 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:24 smithi003 ceph-mon[25122]: purged_snaps scrub ok 2024-06-01T02:20:24.795 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:24 smithi003 ceph-mon[25122]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:24.795 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:24.795 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:24 smithi003 ceph-mon[25122]: osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357] boot 2024-06-01T02:20:24.795 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:24 smithi003 ceph-mon[25122]: osdmap e19: 3 total, 3 up, 3 in 2024-06-01T02:20:24.795 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:24.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:24 smithi027 ceph-mon[27271]: purged_snaps scrub starts 2024-06-01T02:20:24.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:24 smithi027 ceph-mon[27271]: purged_snaps scrub ok 2024-06-01T02:20:24.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:24 smithi027 ceph-mon[27271]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-01T02:20:24.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:24.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:24 smithi027 ceph-mon[27271]: osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357] boot 2024-06-01T02:20:24.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:24 smithi027 ceph-mon[27271]: osdmap e19: 3 total, 3 up, 3 in 2024-06-01T02:20:24.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:20:24.857 INFO:teuthology.orchestra.run.smithi003.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-06-01T02:20:24.857 INFO:teuthology.orchestra.run.smithi003.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-06-01T02:20:24.857 INFO:teuthology.orchestra.run.smithi003.stderr: stderr: 10+0 records in 2024-06-01T02:20:24.857 INFO:teuthology.orchestra.run.smithi003.stderr:10+0 records out 2024-06-01T02:20:24.857 INFO:teuthology.orchestra.run.smithi003.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.013406 s, 782 MB/s 2024-06-01T02:20:24.857 INFO:teuthology.orchestra.run.smithi003.stderr:--> Zapping successful for: 2024-06-01T02:20:25.710 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi003:vg_nvme/lv_1 2024-06-01T02:20:25.942 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:25 smithi003 ceph-mon[25122]: pgmap v67: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:25.943 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:25 smithi003 ceph-mon[25122]: osdmap e20: 3 total, 3 up, 3 in 2024-06-01T02:20:25.943 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:20:25.943 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:20:25.943 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-01T02:20:26.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:25 smithi027 ceph-mon[27271]: pgmap v67: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:26.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:25 smithi027 ceph-mon[27271]: osdmap e20: 3 total, 3 up, 3 in 2024-06-01T02:20:26.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:20:26.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:20:26.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-01T02:20:27.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-01T02:20:27.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:26 smithi003 ceph-mon[25122]: osdmap e21: 3 total, 3 up, 3 in 2024-06-01T02:20:27.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-01T02:20:27.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-01T02:20:27.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:26 smithi027 ceph-mon[27271]: osdmap e21: 3 total, 3 up, 3 in 2024-06-01T02:20:27.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-01T02:20:27.820 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:20:27 smithi003 sudo[43514]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-01T02:20:27.820 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:20:27 smithi003 sudo[43514]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:27.820 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:20:27 smithi003 sudo[43514]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:28.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:27 smithi003 ceph-mon[25122]: pgmap v70: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:28.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:27 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-01T02:20:28.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:27 smithi003 ceph-mon[25122]: osdmap e22: 3 total, 3 up, 3 in 2024-06-01T02:20:28.082 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:20:28 smithi003 sudo[43572]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-01T02:20:28.082 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:20:28 smithi003 sudo[43572]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:28.082 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:20:28 smithi003 sudo[43572]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:28.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:27 smithi027 ceph-mon[27271]: pgmap v70: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:28.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:27 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-01T02:20:28.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:27 smithi027 ceph-mon[27271]: osdmap e22: 3 total, 3 up, 3 in 2024-06-01T02:20:28.798 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:20:28 smithi003 sudo[43635]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-01T02:20:28.798 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:20:28 smithi003 sudo[43635]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:28.799 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:20:28 smithi003 sudo[43635]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:29.049 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:20:28 smithi003 sudo[43736]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-01T02:20:29.049 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:20:28 smithi003 sudo[43736]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:29.049 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:20:28 smithi003 sudo[43736]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:29.049 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:28 smithi003 ceph-mon[25122]: osdmap e23: 3 total, 3 up, 3 in 2024-06-01T02:20:29.050 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:28 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:29.050 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:28 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:29.050 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:28 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:20:29.050 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:28 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:20:29.050 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:28 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:29.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:28 smithi027 ceph-mon[27271]: osdmap e23: 3 total, 3 up, 3 in 2024-06-01T02:20:29.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:28 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:29.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:28 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:29.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:28 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:20:29.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:28 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:20:29.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:28 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:29.747 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:20:29 smithi003 sudo[43787]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-01T02:20:29.747 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:20:29 smithi003 sudo[43787]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:29.747 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:20:29 smithi003 sudo[43787]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:30.048 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:20:29 smithi003 sudo[43810]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-01T02:20:30.048 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:20:29 smithi003 sudo[43810]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:30.048 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:20:29 smithi003 sudo[43810]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:30.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 sudo[43838]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-01T02:20:30.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 sudo[43838]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:30.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 sudo[43838]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:31.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 sudo[43867]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-01T02:20:31.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 sudo[43867]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:31.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 sudo[43867]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:31.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 ceph-mon[25122]: pgmap v73: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:31.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 ceph-mon[25122]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi003:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:20:31.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 ceph-mon[25122]: mgrmap e19: smithi003.njsqxz(active, since 2m), standbys: smithi027.tujskj 2024-06-01T02:20:31.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:30 smithi003 ceph-mon[25122]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-01T02:20:31.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:30 smithi027 ceph-mon[27271]: pgmap v73: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:31.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:30 smithi027 ceph-mon[27271]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi003:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:20:31.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:30 smithi027 ceph-mon[27271]: mgrmap e19: smithi003.njsqxz(active, since 2m), standbys: smithi027.tujskj 2024-06-01T02:20:31.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:30 smithi027 ceph-mon[27271]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-01T02:20:31.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 sudo[29561]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-01T02:20:31.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 sudo[29561]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:31.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 sudo[29561]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:32.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:31 smithi003 ceph-mon[25122]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-01T02:20:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:31 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:20:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:31 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:20:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:31 smithi003 ceph-mon[25122]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-01T02:20:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:31 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:20:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:31 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:20:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:31 smithi003 ceph-mon[25122]: pgmap v74: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:31 smithi003 ceph-mon[25122]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-01T02:20:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:31 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:32.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 sudo[29564]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-01T02:20:32.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 sudo[29564]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-01T02:20:32.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 sudo[29564]: pam_unix(sudo:session): session closed for user root 2024-06-01T02:20:32.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 ceph-mon[27271]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-01T02:20:32.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:20:32.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:20:32.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 ceph-mon[27271]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-01T02:20:32.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:20:32.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:20:32.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 ceph-mon[27271]: pgmap v74: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:32.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 ceph-mon[27271]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-01T02:20:32.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:31 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: Adjusting osd_memory_target on smithi003 to 4289M 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:33.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:33 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:33.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:33.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:33.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:33.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:33.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: Adjusting osd_memory_target on smithi003 to 4289M 2024-06-01T02:20:33.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:33.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:33.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:33.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:33.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:33 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:34.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:34 smithi003 ceph-mon[25122]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:34.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:34 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1063300745' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ae879ca-e6ea-4e19-bdfe-5dcb162ddd30"}]: dispatch 2024-06-01T02:20:34.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:34 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1063300745' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ae879ca-e6ea-4e19-bdfe-5dcb162ddd30"}]': finished 2024-06-01T02:20:34.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:34 smithi003 ceph-mon[25122]: osdmap e24: 4 total, 3 up, 4 in 2024-06-01T02:20:34.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:34 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:34.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:34 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1720103293' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:20:34.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:34 smithi027 ceph-mon[27271]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:34.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:34 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1063300745' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4ae879ca-e6ea-4e19-bdfe-5dcb162ddd30"}]: dispatch 2024-06-01T02:20:34.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:34 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1063300745' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4ae879ca-e6ea-4e19-bdfe-5dcb162ddd30"}]': finished 2024-06-01T02:20:34.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:34 smithi027 ceph-mon[27271]: osdmap e24: 4 total, 3 up, 4 in 2024-06-01T02:20:34.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:34 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:34.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:34 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1720103293' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:20:36.579 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:36 smithi003 ceph-mon[25122]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:36.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:36 smithi027 ceph-mon[27271]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:38.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:38 smithi003 ceph-mon[25122]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-06-01T02:20:38.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:38 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:38.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:38 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:38.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:38 smithi027 ceph-mon[27271]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-06-01T02:20:38.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:38 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:38.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:38 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:40.571 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:40 smithi003 ceph-mon[25122]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-06-01T02:20:40.571 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:40 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-01T02:20:40.571 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:40 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:40.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:40 smithi027 ceph-mon[27271]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-06-01T02:20:40.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:40 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-01T02:20:40.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:40 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:41.491 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:41 smithi003 ceph-mon[25122]: Deploying daemon osd.3 on smithi003 2024-06-01T02:20:41.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:41 smithi027 ceph-mon[27271]: Deploying daemon osd.3 on smithi003 2024-06-01T02:20:42.559 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:42 smithi003 ceph-mon[25122]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-06-01T02:20:42.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:42 smithi027 ceph-mon[27271]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-06-01T02:20:44.512 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:44 smithi003 ceph-mon[25122]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-06-01T02:20:44.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:44 smithi027 ceph-mon[27271]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-06-01T02:20:46.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:46 smithi003 ceph-mon[25122]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-06-01T02:20:46.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:46.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:46.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:46.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:46.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:46 smithi027 ceph-mon[27271]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-06-01T02:20:46.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:46.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:46.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:46.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:47.823 INFO:teuthology.orchestra.run.smithi003.stdout:Created osd(s) 3 on host 'smithi003' 2024-06-01T02:20:48.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:48 smithi003 ceph-mon[25122]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-06-01T02:20:48.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:48 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:48.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:48 smithi027 ceph-mon[27271]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-06-01T02:20:48.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:48 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:48.717 DEBUG:teuthology.orchestra.run.smithi003:osd.3> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.3.service 2024-06-01T02:20:48.719 INFO:tasks.cephadm:Deploying osd.4 on smithi027 with /dev/vg_nvme/lv_4... 2024-06-01T02:20:48.719 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-06-01T02:20:49.324 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:49 smithi027 ceph-mon[27271]: from='osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-01T02:20:49.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:49 smithi003 ceph-mon[25122]: from='osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-01T02:20:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:50 smithi027 ceph-mon[27271]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:50 smithi027 ceph-mon[27271]: from='osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-01T02:20:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:50 smithi027 ceph-mon[27271]: osdmap e25: 4 total, 3 up, 4 in 2024-06-01T02:20:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:50 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:50 smithi027 ceph-mon[27271]: from='osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:20:50.478 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:50 smithi003 ceph-mon[25122]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:50.478 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:50 smithi003 ceph-mon[25122]: from='osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-01T02:20:50.478 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:50 smithi003 ceph-mon[25122]: osdmap e25: 4 total, 3 up, 4 in 2024-06-01T02:20:50.478 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:50 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:50.478 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:50 smithi003 ceph-mon[25122]: from='osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:20:50.479 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:20:50 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:20:50.101+0000 7fc8a98db700 -1 osd.3 0 waiting for initial osdmap 2024-06-01T02:20:50.479 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:20:50 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:20:50.105+0000 7fc8a226e700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:20:50.532 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-06-01T02:20:50.533 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-06-01T02:20:50.533 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-06-01T02:20:50.533 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-06-01T02:20:50.533 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110823 s, 946 MB/s 2024-06-01T02:20:50.533 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-06-01T02:20:51.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_4 2024-06-01T02:20:51.391 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:51 smithi003 ceph-mon[25122]: from='osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]': finished 2024-06-01T02:20:51.391 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:51 smithi003 ceph-mon[25122]: osdmap e26: 4 total, 3 up, 4 in 2024-06-01T02:20:51.391 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:51 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:51.391 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:51 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:51.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:51 smithi027 ceph-mon[27271]: from='osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]': finished 2024-06-01T02:20:51.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:51 smithi027 ceph-mon[27271]: osdmap e26: 4 total, 3 up, 4 in 2024-06-01T02:20:51.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:51 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:51.418 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:51 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:52.378 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:52 smithi003 ceph-mon[25122]: purged_snaps scrub starts 2024-06-01T02:20:52.378 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:52 smithi003 ceph-mon[25122]: purged_snaps scrub ok 2024-06-01T02:20:52.379 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:52 smithi003 ceph-mon[25122]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:52.379 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:52.379 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:52 smithi003 ceph-mon[25122]: osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546] boot 2024-06-01T02:20:52.379 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:52 smithi003 ceph-mon[25122]: osdmap e27: 4 total, 4 up, 4 in 2024-06-01T02:20:52.379 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:52.379 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:52.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:52 smithi027 ceph-mon[27271]: purged_snaps scrub starts 2024-06-01T02:20:52.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:52 smithi027 ceph-mon[27271]: purged_snaps scrub ok 2024-06-01T02:20:52.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:52 smithi027 ceph-mon[27271]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-06-01T02:20:52.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:52.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:52 smithi027 ceph-mon[27271]: osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546] boot 2024-06-01T02:20:52.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:52 smithi027 ceph-mon[27271]: osdmap e27: 4 total, 4 up, 4 in 2024-06-01T02:20:52.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:20:52.415 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:53.302 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:53 smithi027 ceph-mon[27271]: osdmap e28: 4 total, 4 up, 4 in 2024-06-01T02:20:53.302 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:53 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:20:53.302 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:53 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:20:53.302 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:53 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:53.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:53 smithi003 ceph-mon[25122]: osdmap e28: 4 total, 4 up, 4 in 2024-06-01T02:20:53.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:53 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:20:53.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:53 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:20:53.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:53 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:54.344 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:54 smithi027 ceph-mon[27271]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:20:54.345 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:54 smithi027 ceph-mon[27271]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:20:54.513 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:54 smithi003 ceph-mon[25122]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:20:54.513 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:54 smithi003 ceph-mon[25122]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:20:56.186 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: Detected new or changed devices on smithi003 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: Adjusting osd_memory_target on smithi003 to 3217M 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/3717294154' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3a54374-976e-4d5e-9280-30eb479370ba"}]: dispatch 2024-06-01T02:20:56.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3a54374-976e-4d5e-9280-30eb479370ba"}]: dispatch 2024-06-01T02:20:56.188 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3a54374-976e-4d5e-9280-30eb479370ba"}]': finished 2024-06-01T02:20:56.188 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: osdmap e29: 5 total, 4 up, 5 in 2024-06-01T02:20:56.188 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:20:56.188 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:55 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/3878579585' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:20:56.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: Detected new or changed devices on smithi003 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: Adjusting osd_memory_target on smithi003 to 3217M 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/3717294154' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3a54374-976e-4d5e-9280-30eb479370ba"}]: dispatch 2024-06-01T02:20:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3a54374-976e-4d5e-9280-30eb479370ba"}]: dispatch 2024-06-01T02:20:56.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3a54374-976e-4d5e-9280-30eb479370ba"}]': finished 2024-06-01T02:20:56.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: osdmap e29: 5 total, 4 up, 5 in 2024-06-01T02:20:56.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:20:56.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:55 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/3878579585' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:20:58.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:57 smithi003 ceph-mon[25122]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:20:58.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:57 smithi027 ceph-mon[27271]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:00.230 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:59 smithi027 ceph-mon[27271]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:00.231 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:00.231 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:20:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:00.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:59 smithi003 ceph-mon[25122]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:00.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:00.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:20:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:02.232 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:01 smithi027 ceph-mon[27271]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:02.232 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:01 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-01T02:21:02.232 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:01 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:02.232 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:01 smithi027 ceph-mon[27271]: Deploying daemon osd.4 on smithi027 2024-06-01T02:21:02.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:01 smithi003 ceph-mon[25122]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:02.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:01 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-01T02:21:02.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:01 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:02.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:01 smithi003 ceph-mon[25122]: Deploying daemon osd.4 on smithi027 2024-06-01T02:21:04.313 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:03 smithi027 ceph-mon[27271]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:04.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:03 smithi003 ceph-mon[25122]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:06.293 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:05 smithi027 ceph-mon[27271]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:06.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:05 smithi003 ceph-mon[25122]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:07.541 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:07 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:07.541 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:07 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:07.541 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:07 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:07.541 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:07 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:07.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:07 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:07.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:07 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:07.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:07 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:07.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:07 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:08.472 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:08 smithi027 ceph-mon[27271]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:08.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:08 smithi003 ceph-mon[25122]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:09.954 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 4 on host 'smithi027' 2024-06-01T02:21:10.591 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:10 smithi027 ceph-mon[27271]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:10.591 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:10 smithi027 ceph-mon[27271]: from='osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-01T02:21:10.591 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:10.591 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:10.615 DEBUG:teuthology.orchestra.run.smithi027:osd.4> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.4.service 2024-06-01T02:21:10.618 INFO:tasks.cephadm:Deploying osd.5 on smithi027 with /dev/vg_nvme/lv_3... 2024-06-01T02:21:10.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-06-01T02:21:10.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:10 smithi003 ceph-mon[25122]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:10.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:10 smithi003 ceph-mon[25122]: from='osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-01T02:21:10.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:10.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:11.664 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:11 smithi027 ceph-mon[27271]: from='osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-01T02:21:11.664 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:11 smithi027 ceph-mon[27271]: osdmap e30: 5 total, 4 up, 5 in 2024-06-01T02:21:11.665 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:11 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:21:11.665 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:11 smithi027 ceph-mon[27271]: from='osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:11.665 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:21:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:21:11.373+0000 7f5699d11700 -1 osd.4 0 waiting for initial osdmap 2024-06-01T02:21:11.665 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:21:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:21:11.381+0000 7f56966ac700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:21:11.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:11 smithi003 ceph-mon[25122]: from='osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-01T02:21:11.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:11 smithi003 ceph-mon[25122]: osdmap e30: 5 total, 4 up, 5 in 2024-06-01T02:21:11.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:11 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:21:11.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:11 smithi003 ceph-mon[25122]: from='osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:12.503 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:12 smithi027 ceph-mon[27271]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:12.503 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:12 smithi027 ceph-mon[27271]: from='osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-01T02:21:12.504 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:12 smithi027 ceph-mon[27271]: osdmap e31: 5 total, 4 up, 5 in 2024-06-01T02:21:12.504 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:12 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:21:12.504 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:12 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:21:12.504 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:12 smithi027 ceph-mon[27271]: osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855] boot 2024-06-01T02:21:12.504 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:12 smithi027 ceph-mon[27271]: osdmap e32: 5 total, 5 up, 5 in 2024-06-01T02:21:12.504 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:12 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:21:12.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:12 smithi003 ceph-mon[25122]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-01T02:21:12.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:12 smithi003 ceph-mon[25122]: from='osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-01T02:21:12.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:12 smithi003 ceph-mon[25122]: osdmap e31: 5 total, 4 up, 5 in 2024-06-01T02:21:12.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:12 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:21:12.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:12 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:21:12.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:12 smithi003 ceph-mon[25122]: osd.4 [v2:172.21.15.27:6800/3717502855,v1:172.21.15.27:6801/3717502855] boot 2024-06-01T02:21:12.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:12 smithi003 ceph-mon[25122]: osdmap e32: 5 total, 5 up, 5 in 2024-06-01T02:21:12.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:12 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:21:13.553 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:13 smithi027 ceph-mon[27271]: purged_snaps scrub starts 2024-06-01T02:21:13.553 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:13 smithi027 ceph-mon[27271]: purged_snaps scrub ok 2024-06-01T02:21:13.553 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:13 smithi027 ceph-mon[27271]: osdmap e33: 5 total, 5 up, 5 in 2024-06-01T02:21:13.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:13 smithi003 ceph-mon[25122]: purged_snaps scrub starts 2024-06-01T02:21:13.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:13 smithi003 ceph-mon[25122]: purged_snaps scrub ok 2024-06-01T02:21:13.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:13 smithi003 ceph-mon[25122]: osdmap e33: 5 total, 5 up, 5 in 2024-06-01T02:21:13.863 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-06-01T02:21:13.864 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-06-01T02:21:13.864 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-06-01T02:21:13.864 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-06-01T02:21:13.864 INFO:teuthology.orchestra.run.smithi027.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0123556 s, 849 MB/s 2024-06-01T02:21:13.864 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-06-01T02:21:14.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:14 smithi027 ceph-mon[27271]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:14.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:14 smithi027 ceph-mon[27271]: osdmap e34: 5 total, 5 up, 5 in 2024-06-01T02:21:14.819 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:14 smithi003 ceph-mon[25122]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:14.819 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:14 smithi003 ceph-mon[25122]: osdmap e34: 5 total, 5 up, 5 in 2024-06-01T02:21:15.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_3 2024-06-01T02:21:15.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: Detected new or changed devices on smithi027 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: Adjusting osd_memory_target on smithi027 to 15936M 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:15.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:15 smithi003 ceph-mon[25122]: osdmap e35: 5 total, 5 up, 5 in 2024-06-01T02:21:15.917 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: Detected new or changed devices on smithi027 2024-06-01T02:21:15.917 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:15.917 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:21:15.917 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: Adjusting osd_memory_target on smithi027 to 15936M 2024-06-01T02:21:15.918 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:15.918 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:15.918 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:15.918 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:15.918 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:15.918 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:15.918 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:15.918 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:15 smithi027 ceph-mon[27271]: osdmap e35: 5 total, 5 up, 5 in 2024-06-01T02:21:17.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:17 smithi027 ceph-mon[27271]: osdmap e36: 5 total, 5 up, 5 in 2024-06-01T02:21:17.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:17 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:21:17.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:17 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:21:17.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:17 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:17.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:17 smithi003 ceph-mon[25122]: osdmap e36: 5 total, 5 up, 5 in 2024-06-01T02:21:17.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:17 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:21:17.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:17 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:21:17.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:17 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:18.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:18 smithi027 ceph-mon[27271]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-01T02:21:18.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:18 smithi027 ceph-mon[27271]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:21:18.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:18 smithi003 ceph-mon[25122]: pgmap v110: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-06-01T02:21:18.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:18 smithi003 ceph-mon[25122]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:21:20.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:20 smithi003 ceph-mon[25122]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-06-01T02:21:20.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:20 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/647628587' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9bc2fc0-c580-47b3-b37b-91fd8484f319"}]: dispatch 2024-06-01T02:21:20.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:20 smithi003 ceph-mon[25122]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9bc2fc0-c580-47b3-b37b-91fd8484f319"}]: dispatch 2024-06-01T02:21:20.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:20 smithi003 ceph-mon[25122]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e9bc2fc0-c580-47b3-b37b-91fd8484f319"}]': finished 2024-06-01T02:21:20.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:20 smithi003 ceph-mon[25122]: osdmap e37: 6 total, 5 up, 6 in 2024-06-01T02:21:20.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:20 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:20.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:20 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/1935256998' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:21:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:20 smithi027 ceph-mon[27271]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-06-01T02:21:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:20 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/647628587' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9bc2fc0-c580-47b3-b37b-91fd8484f319"}]: dispatch 2024-06-01T02:21:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:20 smithi027 ceph-mon[27271]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9bc2fc0-c580-47b3-b37b-91fd8484f319"}]: dispatch 2024-06-01T02:21:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:20 smithi027 ceph-mon[27271]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e9bc2fc0-c580-47b3-b37b-91fd8484f319"}]': finished 2024-06-01T02:21:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:20 smithi027 ceph-mon[27271]: osdmap e37: 6 total, 5 up, 6 in 2024-06-01T02:21:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:20 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:20.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:20 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/1935256998' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:21:22.777 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:22 smithi027 ceph-mon[27271]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:21:22.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:22 smithi003 ceph-mon[25122]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:21:24.457 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:24 smithi027 ceph-mon[27271]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-06-01T02:21:24.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:24 smithi003 ceph-mon[25122]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-06-01T02:21:25.449 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:21:25.449 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:21:25.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:21:25.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:21:26.585 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:26 smithi027 ceph-mon[27271]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:26.585 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-01T02:21:26.585 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:26.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:26 smithi003 ceph-mon[25122]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:26.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-01T02:21:26.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:27.733 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:27 smithi027 ceph-mon[27271]: Deploying daemon osd.5 on smithi027 2024-06-01T02:21:27.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:27 smithi003 ceph-mon[25122]: Deploying daemon osd.5 on smithi027 2024-06-01T02:21:28.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:28 smithi027 ceph-mon[27271]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:28.831 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:28 smithi003 ceph-mon[25122]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:30.535 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:30 smithi027 ceph-mon[27271]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:30.832 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:30 smithi003 ceph-mon[25122]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:32.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:31 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:32.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:31 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:31 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:31 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:32.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:31 smithi003 ceph-mon[25122]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:32.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:31 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:32.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:31 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:32.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:31 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:32.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:31 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:32.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:31 smithi027 ceph-mon[27271]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:34.148 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:33 smithi027 ceph-mon[27271]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:34.164 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 5 on host 'smithi027' 2024-06-01T02:21:34.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:33 smithi003 ceph-mon[25122]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:35.094 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:34 smithi027 ceph-mon[27271]: from='osd.5 [v2:172.21.15.27:6808/4110125634,v1:172.21.15.27:6809/4110125634]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-01T02:21:35.094 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:34 smithi027 ceph-mon[27271]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-01T02:21:35.094 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:34 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:35.094 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:34 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:35.121 DEBUG:teuthology.orchestra.run.smithi027:osd.5> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.5.service 2024-06-01T02:21:35.123 INFO:tasks.cephadm:Deploying osd.6 on smithi027 with /dev/vg_nvme/lv_2... 2024-06-01T02:21:35.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-06-01T02:21:35.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:34 smithi003 ceph-mon[25122]: from='osd.5 [v2:172.21.15.27:6808/4110125634,v1:172.21.15.27:6809/4110125634]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-01T02:21:35.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:34 smithi003 ceph-mon[25122]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-01T02:21:35.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:34 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:35.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:34 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:36.185 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:35 smithi027 ceph-mon[27271]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:36.185 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:35 smithi027 ceph-mon[27271]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-01T02:21:36.185 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:35 smithi027 ceph-mon[27271]: osdmap e38: 6 total, 5 up, 6 in 2024-06-01T02:21:36.185 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:35 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:36.185 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:35 smithi027 ceph-mon[27271]: from='osd.5 [v2:172.21.15.27:6808/4110125634,v1:172.21.15.27:6809/4110125634]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:36.185 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:35 smithi027 ceph-mon[27271]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:36.186 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:21:35 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:21:35.918+0000 7fe2a1a3c700 -1 osd.5 0 waiting for initial osdmap 2024-06-01T02:21:36.186 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:21:35 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:21:35.923+0000 7fe29cbd4700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:21:36.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:35 smithi003 ceph-mon[25122]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:36.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:35 smithi003 ceph-mon[25122]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-01T02:21:36.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:35 smithi003 ceph-mon[25122]: osdmap e38: 6 total, 5 up, 6 in 2024-06-01T02:21:36.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:35 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:36.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:35 smithi003 ceph-mon[25122]: from='osd.5 [v2:172.21.15.27:6808/4110125634,v1:172.21.15.27:6809/4110125634]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:36.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:35 smithi003 ceph-mon[25122]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:37.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:36 smithi027 ceph-mon[27271]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-01T02:21:37.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:36 smithi027 ceph-mon[27271]: osdmap e39: 6 total, 5 up, 6 in 2024-06-01T02:21:37.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:36 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:37.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:36 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:37.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:36 smithi027 ceph-mon[27271]: osd.5 [v2:172.21.15.27:6808/4110125634,v1:172.21.15.27:6809/4110125634] boot 2024-06-01T02:21:37.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:36 smithi027 ceph-mon[27271]: osdmap e40: 6 total, 6 up, 6 in 2024-06-01T02:21:37.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:36 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:37.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:36 smithi003 ceph-mon[25122]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-01T02:21:37.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:36 smithi003 ceph-mon[25122]: osdmap e39: 6 total, 5 up, 6 in 2024-06-01T02:21:37.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:36 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:37.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:36 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:37.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:36 smithi003 ceph-mon[25122]: osd.5 [v2:172.21.15.27:6808/4110125634,v1:172.21.15.27:6809/4110125634] boot 2024-06-01T02:21:37.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:36 smithi003 ceph-mon[25122]: osdmap e40: 6 total, 6 up, 6 in 2024-06-01T02:21:37.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:36 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:21:37.450 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-06-01T02:21:37.450 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-06-01T02:21:37.451 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-06-01T02:21:37.451 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-06-01T02:21:37.451 INFO:teuthology.orchestra.run.smithi027.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0120666 s, 869 MB/s 2024-06-01T02:21:37.451 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-06-01T02:21:38.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:37 smithi027 ceph-mon[27271]: purged_snaps scrub starts 2024-06-01T02:21:38.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:37 smithi027 ceph-mon[27271]: purged_snaps scrub ok 2024-06-01T02:21:38.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:37 smithi027 ceph-mon[27271]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:38.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:37 smithi027 ceph-mon[27271]: osdmap e41: 6 total, 6 up, 6 in 2024-06-01T02:21:38.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_2 2024-06-01T02:21:38.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:37 smithi003 ceph-mon[25122]: purged_snaps scrub starts 2024-06-01T02:21:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:37 smithi003 ceph-mon[25122]: purged_snaps scrub ok 2024-06-01T02:21:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:37 smithi003 ceph-mon[25122]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-01T02:21:38.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:37 smithi003 ceph-mon[25122]: osdmap e41: 6 total, 6 up, 6 in 2024-06-01T02:21:40.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:40.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: osdmap e42: 6 total, 6 up, 6 in 2024-06-01T02:21:40.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:40.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:21:40.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:21:40.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:40.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:40.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:40.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:40.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:40.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:39 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:40.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: osdmap e42: 6 total, 6 up, 6 in 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:40.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:39 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:41.313 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:40 smithi027 ceph-mon[27271]: Detected new or changed devices on smithi027 2024-06-01T02:21:41.313 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:40 smithi027 ceph-mon[27271]: Adjusting osd_memory_target on smithi027 to 7968M 2024-06-01T02:21:41.313 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:40 smithi027 ceph-mon[27271]: osdmap e43: 6 total, 6 up, 6 in 2024-06-01T02:21:41.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:40 smithi003 ceph-mon[25122]: Detected new or changed devices on smithi027 2024-06-01T02:21:41.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:40 smithi003 ceph-mon[25122]: Adjusting osd_memory_target on smithi027 to 7968M 2024-06-01T02:21:41.334 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:40 smithi003 ceph-mon[25122]: osdmap e43: 6 total, 6 up, 6 in 2024-06-01T02:21:42.245 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:41 smithi027 ceph-mon[27271]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:42.245 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:41 smithi027 ceph-mon[27271]: osdmap e44: 6 total, 6 up, 6 in 2024-06-01T02:21:42.246 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:41 smithi027 ceph-mon[27271]: from='client.24185 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:21:42.246 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:21:42.246 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:21:42.246 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:41 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:42.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:41 smithi003 ceph-mon[25122]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:42.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:41 smithi003 ceph-mon[25122]: osdmap e44: 6 total, 6 up, 6 in 2024-06-01T02:21:42.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:41 smithi003 ceph-mon[25122]: from='client.24185 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:21:42.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:41 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:21:42.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:41 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:21:42.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:41 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:44.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:43 smithi027 ceph-mon[27271]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:44.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:43 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/2396290529' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8011a9c-5bde-4bd4-8ca5-c2885410df0b"}]: dispatch 2024-06-01T02:21:44.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:43 smithi027 ceph-mon[27271]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8011a9c-5bde-4bd4-8ca5-c2885410df0b"}]: dispatch 2024-06-01T02:21:44.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:43 smithi027 ceph-mon[27271]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8011a9c-5bde-4bd4-8ca5-c2885410df0b"}]': finished 2024-06-01T02:21:44.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:43 smithi027 ceph-mon[27271]: osdmap e45: 7 total, 6 up, 7 in 2024-06-01T02:21:44.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:21:44.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:43 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/707243201' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:21:44.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:43 smithi003 ceph-mon[25122]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:44.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:43 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/2396290529' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8011a9c-5bde-4bd4-8ca5-c2885410df0b"}]: dispatch 2024-06-01T02:21:44.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:43 smithi003 ceph-mon[25122]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8011a9c-5bde-4bd4-8ca5-c2885410df0b"}]: dispatch 2024-06-01T02:21:44.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:43 smithi003 ceph-mon[25122]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8011a9c-5bde-4bd4-8ca5-c2885410df0b"}]': finished 2024-06-01T02:21:44.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:43 smithi003 ceph-mon[25122]: osdmap e45: 7 total, 6 up, 7 in 2024-06-01T02:21:44.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:21:44.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:43 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/707243201' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:21:46.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:45 smithi003 ceph-mon[25122]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:46.337 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:45 smithi027 ceph-mon[27271]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:48.083 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:47 smithi027 ceph-mon[27271]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:48.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:47 smithi003 ceph-mon[25122]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:50.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:49 smithi003 ceph-mon[25122]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:50.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:49 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-01T02:21:50.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:49 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:50.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:49 smithi003 ceph-mon[25122]: Deploying daemon osd.6 on smithi027 2024-06-01T02:21:50.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:49 smithi027 ceph-mon[27271]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:49 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-01T02:21:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:49 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:49 smithi027 ceph-mon[27271]: Deploying daemon osd.6 on smithi027 2024-06-01T02:21:52.274 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:51 smithi027 ceph-mon[27271]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:52.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:51 smithi003 ceph-mon[25122]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:54.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:53 smithi003 ceph-mon[25122]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:54.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:53 smithi027 ceph-mon[27271]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:55.314 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:55.314 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:55.314 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:55.314 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:55.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:55.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:21:55.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:21:55.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:21:56.128 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:56 smithi027 ceph-mon[27271]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:56.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:56 smithi003 ceph-mon[25122]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:57.559 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 6 on host 'smithi027' 2024-06-01T02:21:58.223 DEBUG:teuthology.orchestra.run.smithi027:osd.6> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.6.service 2024-06-01T02:21:58.226 INFO:tasks.cephadm:Deploying osd.7 on smithi027 with /dev/vg_nvme/lv_1... 2024-06-01T02:21:58.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-06-01T02:21:58.290 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:58 smithi027 ceph-mon[27271]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:58.291 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:58 smithi027 ceph-mon[27271]: from='osd.6 [v2:172.21.15.27:6816/1713832118,v1:172.21.15.27:6817/1713832118]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-01T02:21:58.291 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:58 smithi027 ceph-mon[27271]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-01T02:21:58.291 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:58.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:58 smithi003 ceph-mon[25122]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:21:58.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:58 smithi003 ceph-mon[25122]: from='osd.6 [v2:172.21.15.27:6816/1713832118,v1:172.21.15.27:6817/1713832118]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-01T02:21:58.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:58 smithi003 ceph-mon[25122]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-01T02:21:58.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:59.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:59 smithi003 ceph-mon[25122]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-01T02:21:59.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:59 smithi003 ceph-mon[25122]: osdmap e46: 7 total, 6 up, 7 in 2024-06-01T02:21:59.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:21:59.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:59 smithi003 ceph-mon[25122]: from='osd.6 [v2:172.21.15.27:6816/1713832118,v1:172.21.15.27:6817/1713832118]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:59.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:59 smithi003 ceph-mon[25122]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:59.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:21:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:59.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:59 smithi027 ceph-mon[27271]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-01T02:21:59.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:59 smithi027 ceph-mon[27271]: osdmap e46: 7 total, 6 up, 7 in 2024-06-01T02:21:59.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:21:59.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:59 smithi027 ceph-mon[27271]: from='osd.6 [v2:172.21.15.27:6816/1713832118,v1:172.21.15.27:6817/1713832118]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:59.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:59 smithi027 ceph-mon[27271]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:21:59.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:21:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:21:59.348 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:21:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[37402]: 2024-06-01T02:21:59.023+0000 7fb324c1a700 -1 osd.6 0 waiting for initial osdmap 2024-06-01T02:21:59.348 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:21:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[37402]: 2024-06-01T02:21:59.030+0000 7fb31f5b1700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:22:00.278 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:00 smithi027 ceph-mon[27271]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:22:00.278 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:00 smithi027 ceph-mon[27271]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-01T02:22:00.278 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:00 smithi027 ceph-mon[27271]: osdmap e47: 7 total, 6 up, 7 in 2024-06-01T02:22:00.278 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:00 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:22:00.278 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:00 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:22:00.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:00 smithi003 ceph-mon[25122]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:22:00.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:00 smithi003 ceph-mon[25122]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-01T02:22:00.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:00 smithi003 ceph-mon[25122]: osdmap e47: 7 total, 6 up, 7 in 2024-06-01T02:22:00.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:00 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:22:00.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:00 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:22:00.729 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-06-01T02:22:00.729 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-06-01T02:22:00.729 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-06-01T02:22:00.729 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-06-01T02:22:00.729 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122664 s, 855 MB/s 2024-06-01T02:22:00.729 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-06-01T02:22:01.201 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:01 smithi027 ceph-mon[27271]: purged_snaps scrub starts 2024-06-01T02:22:01.201 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:01 smithi027 ceph-mon[27271]: purged_snaps scrub ok 2024-06-01T02:22:01.202 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:01 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:22:01.202 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:01 smithi027 ceph-mon[27271]: osd.6 [v2:172.21.15.27:6816/1713832118,v1:172.21.15.27:6817/1713832118] boot 2024-06-01T02:22:01.202 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:01 smithi027 ceph-mon[27271]: osdmap e48: 7 total, 7 up, 7 in 2024-06-01T02:22:01.202 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:01 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:22:01.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:01 smithi003 ceph-mon[25122]: purged_snaps scrub starts 2024-06-01T02:22:01.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:01 smithi003 ceph-mon[25122]: purged_snaps scrub ok 2024-06-01T02:22:01.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:01 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:22:01.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:01 smithi003 ceph-mon[25122]: osd.6 [v2:172.21.15.27:6816/1713832118,v1:172.21.15.27:6817/1713832118] boot 2024-06-01T02:22:01.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:01 smithi003 ceph-mon[25122]: osdmap e48: 7 total, 7 up, 7 in 2024-06-01T02:22:01.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:01 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:22:01.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_1 2024-06-01T02:22:02.055 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:02 smithi027 ceph-mon[27271]: pgmap v144: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:22:02.056 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:02 smithi027 ceph-mon[27271]: osdmap e49: 7 total, 7 up, 7 in 2024-06-01T02:22:02.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:02 smithi003 ceph-mon[25122]: pgmap v144: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-01T02:22:02.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:02 smithi003 ceph-mon[25122]: osdmap e49: 7 total, 7 up, 7 in 2024-06-01T02:22:03.129 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:03 smithi027 ceph-mon[27271]: osdmap e50: 7 total, 7 up, 7 in 2024-06-01T02:22:03.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:03 smithi003 ceph-mon[25122]: osdmap e50: 7 total, 7 up, 7 in 2024-06-01T02:22:04.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:04.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: osdmap e51: 7 total, 7 up, 7 in 2024-06-01T02:22:04.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: Detected new or changed devices on smithi027 2024-06-01T02:22:04.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: Adjusting osd_memory_target on smithi027 to 5312M 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:22:04.099 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:04.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:04.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: osdmap e51: 7 total, 7 up, 7 in 2024-06-01T02:22:04.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: Detected new or changed devices on smithi027 2024-06-01T02:22:04.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:04.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:04.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:04.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:04.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: Adjusting osd_memory_target on smithi027 to 5312M 2024-06-01T02:22:04.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:04.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:04.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:22:04.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:04.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:22:04.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:05.282 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:05 smithi027 ceph-mon[27271]: osdmap e52: 7 total, 7 up, 7 in 2024-06-01T02:22:05.282 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:05 smithi027 ceph-mon[27271]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:22:05.283 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:22:05.283 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:22:05.283 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:05.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:05 smithi003 ceph-mon[25122]: osdmap e52: 7 total, 7 up, 7 in 2024-06-01T02:22:05.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:05 smithi003 ceph-mon[25122]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:22:05.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-01T02:22:05.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-01T02:22:05.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:06.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:06 smithi003 ceph-mon[25122]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:06.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:06 smithi027 ceph-mon[27271]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:07.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:07 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/2382327950' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22d2a92a-4c2d-4a00-a8b8-57d17327fa9a"}]: dispatch 2024-06-01T02:22:07.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:07 smithi003 ceph-mon[25122]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22d2a92a-4c2d-4a00-a8b8-57d17327fa9a"}]: dispatch 2024-06-01T02:22:07.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:07 smithi003 ceph-mon[25122]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "22d2a92a-4c2d-4a00-a8b8-57d17327fa9a"}]': finished 2024-06-01T02:22:07.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:07 smithi003 ceph-mon[25122]: osdmap e53: 8 total, 7 up, 8 in 2024-06-01T02:22:07.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:07 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:07.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:07 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/2382327950' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22d2a92a-4c2d-4a00-a8b8-57d17327fa9a"}]: dispatch 2024-06-01T02:22:07.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:07 smithi027 ceph-mon[27271]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22d2a92a-4c2d-4a00-a8b8-57d17327fa9a"}]: dispatch 2024-06-01T02:22:07.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:07 smithi027 ceph-mon[27271]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "22d2a92a-4c2d-4a00-a8b8-57d17327fa9a"}]': finished 2024-06-01T02:22:07.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:07 smithi027 ceph-mon[27271]: osdmap e53: 8 total, 7 up, 8 in 2024-06-01T02:22:07.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:07 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:08.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:08 smithi003 ceph-mon[25122]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-06-01T02:22:08.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:08 smithi003 ceph-mon[25122]: from='client.? 172.21.15.27:0/1349412742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:22:08.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:08 smithi027 ceph-mon[27271]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-06-01T02:22:08.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:08 smithi027 ceph-mon[27271]: from='client.? 172.21.15.27:0/1349412742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-01T02:22:10.138 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:10 smithi027 ceph-mon[27271]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:22:10.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:10 smithi003 ceph-mon[25122]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:22:12.316 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:12 smithi027 ceph-mon[27271]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-06-01T02:22:12.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:12 smithi003 ceph-mon[25122]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-06-01T02:22:13.281 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:13 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-01T02:22:13.281 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:13 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:13.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:13 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-01T02:22:13.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:13 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:14.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:14 smithi003 ceph-mon[25122]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-06-01T02:22:14.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:14 smithi003 ceph-mon[25122]: Deploying daemon osd.7 on smithi027 2024-06-01T02:22:14.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:14 smithi027 ceph-mon[27271]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-06-01T02:22:14.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:14 smithi027 ceph-mon[27271]: Deploying daemon osd.7 on smithi027 2024-06-01T02:22:16.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:16 smithi027 ceph-mon[27271]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-01T02:22:16.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:16 smithi003 ceph-mon[25122]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-01T02:22:18.137 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:18 smithi027 ceph-mon[27271]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:18.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:18 smithi003 ceph-mon[25122]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:19.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:19 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:19.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:19 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:22:19.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:19 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:19.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:19 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:22:19.486 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:19 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:19.487 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:19 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:22:19.487 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:19 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:19.487 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:19 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:22:20.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:20 smithi003 ceph-mon[25122]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:20.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:20 smithi027 ceph-mon[27271]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:21.367 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 7 on host 'smithi027' 2024-06-01T02:22:21.922 DEBUG:teuthology.orchestra.run.smithi027:osd.7> sudo journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.7.service 2024-06-01T02:22:21.924 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-06-01T02:22:21.925 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd stat -f json 2024-06-01T02:22:22.248 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:22 smithi003 ceph-mon[25122]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:22.248 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:22 smithi003 ceph-mon[25122]: from='osd.7 [v2:172.21.15.27:6824/3680333329,v1:172.21.15.27:6825/3680333329]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-01T02:22:22.248 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:22 smithi003 ceph-mon[25122]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-01T02:22:22.249 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:22 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:22.318 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:22 smithi027 ceph-mon[27271]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:22.318 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:22 smithi027 ceph-mon[27271]: from='osd.7 [v2:172.21.15.27:6824/3680333329,v1:172.21.15.27:6825/3680333329]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-01T02:22:22.318 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:22 smithi027 ceph-mon[27271]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-01T02:22:22.318 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:22 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:23.479 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:23 smithi027 ceph-mon[27271]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-01T02:22:23.479 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:23 smithi027 ceph-mon[27271]: osdmap e54: 8 total, 7 up, 8 in 2024-06-01T02:22:23.479 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:23 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:23.479 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:23 smithi027 ceph-mon[27271]: from='osd.7 [v2:172.21.15.27:6824/3680333329,v1:172.21.15.27:6825/3680333329]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:22:23.479 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:23 smithi027 ceph-mon[27271]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:22:23.479 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:23 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:23.479 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:22:23 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[40459]: 2024-06-01T02:22:23.172+0000 7fd5589f7700 -1 osd.7 0 waiting for initial osdmap 2024-06-01T02:22:23.479 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:22:23 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[40459]: 2024-06-01T02:22:23.180+0000 7fd55138a700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:22:23.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:23 smithi003 ceph-mon[25122]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-01T02:22:23.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:23 smithi003 ceph-mon[25122]: osdmap e54: 8 total, 7 up, 8 in 2024-06-01T02:22:23.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:23 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:23.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:23 smithi003 ceph-mon[25122]: from='osd.7 [v2:172.21.15.27:6824/3680333329,v1:172.21.15.27:6825/3680333329]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:22:23.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:23 smithi003 ceph-mon[25122]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:22:23.506 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:23 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:23.918 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:22:24.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:24 smithi027 ceph-mon[27271]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:24.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:24 smithi027 ceph-mon[27271]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-01T02:22:24.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:24 smithi027 ceph-mon[27271]: osdmap e55: 8 total, 7 up, 8 in 2024-06-01T02:22:24.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:24.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:24.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:24 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1606914198' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-01T02:22:24.557 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:24 smithi003 ceph-mon[25122]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:24.557 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:24 smithi003 ceph-mon[25122]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-06-01T02:22:24.557 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:24 smithi003 ceph-mon[25122]: osdmap e55: 8 total, 7 up, 8 in 2024-06-01T02:22:24.557 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:24.557 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:24.557 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:24 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1606914198' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-01T02:22:24.585 INFO:teuthology.orchestra.run.smithi003.stdout:{"epoch":55,"num_osds":8,"num_up_osds":7,"osd_up_since":1717208520,"num_in_osds":8,"osd_in_since":1717208526,"num_remapped_pgs":0} 2024-06-01T02:22:25.514 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:25 smithi027 ceph-mon[27271]: purged_snaps scrub starts 2024-06-01T02:22:25.515 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:25 smithi027 ceph-mon[27271]: purged_snaps scrub ok 2024-06-01T02:22:25.515 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:25 smithi027 ceph-mon[27271]: osd.7 [v2:172.21.15.27:6824/3680333329,v1:172.21.15.27:6825/3680333329] boot 2024-06-01T02:22:25.515 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:25 smithi027 ceph-mon[27271]: osdmap e56: 8 total, 8 up, 8 in 2024-06-01T02:22:25.515 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:25.515 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:22:25.515 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:22:25.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:25 smithi003 ceph-mon[25122]: purged_snaps scrub starts 2024-06-01T02:22:25.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:25 smithi003 ceph-mon[25122]: purged_snaps scrub ok 2024-06-01T02:22:25.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:25 smithi003 ceph-mon[25122]: osd.7 [v2:172.21.15.27:6824/3680333329,v1:172.21.15.27:6825/3680333329] boot 2024-06-01T02:22:25.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:25 smithi003 ceph-mon[25122]: osdmap e56: 8 total, 8 up, 8 in 2024-06-01T02:22:25.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:22:25.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:22:25.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:22:25.586 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd stat -f json 2024-06-01T02:22:26.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: osdmap e57: 8 total, 8 up, 8 in 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:26.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:22:26.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:26.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-06-01T02:22:26.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: osdmap e57: 8 total, 8 up, 8 in 2024-06-01T02:22:26.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:26.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:22:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:22:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:22:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:22:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:22:27.565 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:22:27.565 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:27 smithi003 ceph-mon[25122]: Detected new or changed devices on smithi027 2024-06-01T02:22:27.566 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:27 smithi003 ceph-mon[25122]: Adjusting osd_memory_target on smithi027 to 3984M 2024-06-01T02:22:27.566 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:27 smithi003 ceph-mon[25122]: osdmap e58: 8 total, 8 up, 8 in 2024-06-01T02:22:27.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:27 smithi027 ceph-mon[27271]: Detected new or changed devices on smithi027 2024-06-01T02:22:27.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:27 smithi027 ceph-mon[27271]: Adjusting osd_memory_target on smithi027 to 3984M 2024-06-01T02:22:27.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:27 smithi027 ceph-mon[27271]: osdmap e58: 8 total, 8 up, 8 in 2024-06-01T02:22:28.111 INFO:teuthology.orchestra.run.smithi003.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1717208544,"num_in_osds":8,"osd_in_since":1717208526,"num_remapped_pgs":0} 2024-06-01T02:22:28.112 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd dump --format=json 2024-06-01T02:22:28.338 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:28.461 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:28 smithi003 ceph-mon[25122]: pgmap v167: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:28.462 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:28 smithi003 ceph-mon[25122]: osdmap e59: 8 total, 8 up, 8 in 2024-06-01T02:22:28.462 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:28 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3730750622' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-01T02:22:28.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:28 smithi027 ceph-mon[27271]: pgmap v167: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:28.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:28 smithi027 ceph-mon[27271]: osdmap e59: 8 total, 8 up, 8 in 2024-06-01T02:22:28.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:28 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/3730750622' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-01T02:22:29.460 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:29 smithi003 ceph-mon[25122]: osdmap e60: 8 total, 8 up, 8 in 2024-06-01T02:22:29.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:29 smithi027 ceph-mon[27271]: osdmap e60: 8 total, 8 up, 8 in 2024-06-01T02:22:30.119 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:22:30.120 INFO:teuthology.orchestra.run.smithi003.stdout:{"epoch":60,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","created":"2024-06-01T02:16:37.976034+0000","modified":"2024-06-01T02:22:28.191086+0000","last_up_change":"2024-06-01T02:22:24.170381+0000","last_in_change":"2024-06-01T02:22:06.674379+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-01T02:20:25.313034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fd50417e-a988-4f23-87d1-e45084b46b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6802","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6803","nonce":2738410731}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6804","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6805","nonce":2738410731}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6808","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6809","nonce":2738410731}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6806","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6807","nonce":2738410731}]},"public_addr":"172.21.15.3:6803/2738410731","cluster_addr":"172.21.15.3:6805/2738410731","heartbeat_back_addr":"172.21.15.3:6809/2738410731","heartbeat_front_addr":"172.21.15.3:6807/2738410731","state":["exists","up"]},{"osd":1,"uuid":"abe11909-0ec2-46d5-9dcb-2234cfbf5e1c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6810","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6811","nonce":2291814156}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6812","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6813","nonce":2291814156}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6816","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6817","nonce":2291814156}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6814","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6815","nonce":2291814156}]},"public_addr":"172.21.15.3:6811/2291814156","cluster_addr":"172.21.15.3:6813/2291814156","heartbeat_back_addr":"172.21.15.3:6817/2291814156","heartbeat_front_addr":"172.21.15.3:6815/2291814156","state":["exists","up"]},{"osd":2,"uuid":"02170894-eb8b-446e-b3be-72db8229c089","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6818","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6819","nonce":2180786357}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6820","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6821","nonce":2180786357}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6824","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6825","nonce":2180786357}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6822","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6823","nonce":2180786357}]},"public_addr":"172.21.15.3:6819/2180786357","cluster_addr":"172.21.15.3:6821/2180786357","heartbeat_back_addr":"172.21.15.3:6825/2180786357","heartbeat_front_addr":"172.21.15.3:6823/2180786357","state":["exists","up"]},{"osd":3,"uuid":"4ae879ca-e6ea-4e19-bdfe-5dcb162ddd30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6826","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6827","nonce":1979857546}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6828","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6829","nonce":1979857546}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6832","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6833","nonce":1979857546}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6830","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6831","nonce":1979857546}]},"public_addr":"172.21.15.3:6827/1979857546","cluster_addr":"172.21.15.3:6829/1979857546","heartbeat_back_addr":"172.21.15.3:6833/1979857546","heartbeat_front_addr":"172.21.15.3:6831/1979857546","state":["exists","up"]},{"osd":4,"uuid":"a3a54374-976e-4d5e-9280-30eb479370ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6801","nonce":3717502855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6803","nonce":3717502855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6807","nonce":3717502855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6805","nonce":3717502855}]},"public_addr":"172.21.15.27:6801/3717502855","cluster_addr":"172.21.15.27:6803/3717502855","heartbeat_back_addr":"172.21.15.27:6807/3717502855","heartbeat_front_addr":"172.21.15.27:6805/3717502855","state":["exists","up"]},{"osd":5,"uuid":"e9bc2fc0-c580-47b3-b37b-91fd8484f319","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6809","nonce":4110125634}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6811","nonce":4110125634}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6815","nonce":4110125634}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6813","nonce":4110125634}]},"public_addr":"172.21.15.27:6809/4110125634","cluster_addr":"172.21.15.27:6811/4110125634","heartbeat_back_addr":"172.21.15.27:6815/4110125634","heartbeat_front_addr":"172.21.15.27:6813/4110125634","state":["exists","up"]},{"osd":6,"uuid":"b8011a9c-5bde-4bd4-8ca5-c2885410df0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6817","nonce":1713832118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6819","nonce":1713832118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6823","nonce":1713832118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6821","nonce":1713832118}]},"public_addr":"172.21.15.27:6817/1713832118","cluster_addr":"172.21.15.27:6819/1713832118","heartbeat_back_addr":"172.21.15.27:6823/1713832118","heartbeat_front_addr":"172.21.15.27:6821/1713832118","state":["exists","up"]},{"osd":7,"uuid":"22d2a92a-4c2d-4a00-a8b8-57d17327fa9a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6825","nonce":3680333329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6827","nonce":3680333329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6831","nonce":3680333329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6829","nonce":3680333329}]},"public_addr":"172.21.15.27:6825/3680333329","cluster_addr":"172.21.15.27:6827/3680333329","heartbeat_back_addr":"172.21.15.27:6831/3680333329","heartbeat_front_addr":"172.21.15.27:6829/3680333329","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:19:23.051172+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:19:53.134218+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:20:21.880355+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:20:49.239590+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:21:10.456027+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:21:34.878758+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:21:58.113324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:22:22.192021+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.3:0/983880676":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/2300598431":"2024-06-02T02:18:24.800609+0000","172.21.15.3:6801/4180696651":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/2960826803":"2024-06-02T02:17:31.096287+0000","172.21.15.3:6801/4010164652":"2024-06-02T02:17:31.096287+0000","172.21.15.3:6800/4010164652":"2024-06-02T02:17:31.096287+0000","172.21.15.3:0/2588496657":"2024-06-02T02:17:31.096287+0000","172.21.15.3:0/2551380483":"2024-06-02T02:17:02.369055+0000","172.21.15.3:6800/4180696651":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/3407438145":"2024-06-02T02:17:31.096287+0000","172.21.15.3:0/378330486":"2024-06-02T02:17:02.369055+0000","172.21.15.3:6800/169228208":"2024-06-02T02:17:02.369055+0000","172.21.15.3:6801/169228208":"2024-06-02T02:17:02.369055+0000","172.21.15.3:0/3124197876":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/3772716787":"2024-06-02T02:17:02.369055+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-01T02:22:30.475 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:30 smithi003 ceph-mon[25122]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:30.475 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:30 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1998195424' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-01T02:22:30.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:30 smithi027 ceph-mon[27271]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:30.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:30 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1998195424' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-01T02:22:30.786 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-06-01T02:20:25.313034+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-06-01T02:22:30.786 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-06-01T02:22:31.013 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:32.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:32 smithi003 ceph-mon[25122]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-06-01T02:22:32.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:32 smithi027 ceph-mon[27271]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-06-01T02:22:32.998 INFO:teuthology.orchestra.run.smithi003.stdout:pg_num: 1 2024-06-01T02:22:33.564 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:33 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/4156237364' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-01T02:22:33.594 INFO:tasks.cephadm:Setting up client nodes... 2024-06-01T02:22:33.594 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-06-01T02:22:33.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:33 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/4156237364' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-01T02:22:34.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:34 smithi003 ceph-mon[25122]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-06-01T02:22:34.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:34 smithi027 ceph-mon[27271]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-06-01T02:22:35.605 INFO:teuthology.orchestra.run.smithi003.stdout:[client.0] 2024-06-01T02:22:35.605 INFO:teuthology.orchestra.run.smithi003.stdout: key = AQDrhVpmKgsaJBAA/tD4QsfsyeukSEfcnDLWDg== 2024-06-01T02:22:36.311 DEBUG:teuthology.orchestra.run.smithi003:> set -ex 2024-06-01T02:22:36.312 DEBUG:teuthology.orchestra.run.smithi003:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-06-01T02:22:36.312 DEBUG:teuthology.orchestra.run.smithi003:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-06-01T02:22:36.352 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-06-01T02:22:36.352 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-06-01T02:22:36.353 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph mgr dump --format=json 2024-06-01T02:22:36.568 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:36 smithi003 ceph-mon[25122]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-01T02:22:36.569 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:36 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2241754944' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-01T02:22:36.569 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:36 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2241754944' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-01T02:22:36.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:36 smithi027 ceph-mon[27271]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-06-01T02:22:36.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:36 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/2241754944' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-01T02:22:36.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:36 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/2241754944' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-01T02:22:36.619 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:38.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:38 smithi003 ceph-mon[25122]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-06-01T02:22:38.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:38 smithi027 ceph-mon[27271]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-06-01T02:22:38.609 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:22:39.338 INFO:teuthology.orchestra.run.smithi003.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi003.njsqxz","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6800","nonce":4060740612},{"type":"v1","addr":"172.21.15.3:6801","nonce":4060740612}]},"active_addr":"172.21.15.3:6801/4060740612","active_change":"2024-06-01T02:18:24.800983+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi027.tujskj","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.3:8443/","prometheus":"http://172.21.15.3:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.3:0","nonce":2865133815}]},{"addrvec":[{"type":"v2","addr":"172.21.15.3:0","nonce":3871483399}]},{"addrvec":[{"type":"v2","addr":"172.21.15.3:0","nonce":3191027430}]},{"addrvec":[{"type":"v2","addr":"172.21.15.3:0","nonce":2909994409}]}]}} 2024-06-01T02:22:39.343 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-06-01T02:22:39.343 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-06-01T02:22:39.344 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd dump --format=json 2024-06-01T02:22:39.575 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:39.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:39 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1459487493' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-01T02:22:39.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:39 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1459487493' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-01T02:22:40.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:40 smithi003 ceph-mon[25122]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-06-01T02:22:40.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:40 smithi027 ceph-mon[27271]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-06-01T02:22:41.413 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:22:41.414 INFO:teuthology.orchestra.run.smithi003.stdout:{"epoch":60,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","created":"2024-06-01T02:16:37.976034+0000","modified":"2024-06-01T02:22:28.191086+0000","last_up_change":"2024-06-01T02:22:24.170381+0000","last_in_change":"2024-06-01T02:22:06.674379+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-01T02:20:25.313034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fd50417e-a988-4f23-87d1-e45084b46b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6802","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6803","nonce":2738410731}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6804","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6805","nonce":2738410731}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6808","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6809","nonce":2738410731}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6806","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6807","nonce":2738410731}]},"public_addr":"172.21.15.3:6803/2738410731","cluster_addr":"172.21.15.3:6805/2738410731","heartbeat_back_addr":"172.21.15.3:6809/2738410731","heartbeat_front_addr":"172.21.15.3:6807/2738410731","state":["exists","up"]},{"osd":1,"uuid":"abe11909-0ec2-46d5-9dcb-2234cfbf5e1c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6810","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6811","nonce":2291814156}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6812","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6813","nonce":2291814156}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6816","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6817","nonce":2291814156}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6814","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6815","nonce":2291814156}]},"public_addr":"172.21.15.3:6811/2291814156","cluster_addr":"172.21.15.3:6813/2291814156","heartbeat_back_addr":"172.21.15.3:6817/2291814156","heartbeat_front_addr":"172.21.15.3:6815/2291814156","state":["exists","up"]},{"osd":2,"uuid":"02170894-eb8b-446e-b3be-72db8229c089","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6818","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6819","nonce":2180786357}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6820","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6821","nonce":2180786357}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6824","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6825","nonce":2180786357}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6822","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6823","nonce":2180786357}]},"public_addr":"172.21.15.3:6819/2180786357","cluster_addr":"172.21.15.3:6821/2180786357","heartbeat_back_addr":"172.21.15.3:6825/2180786357","heartbeat_front_addr":"172.21.15.3:6823/2180786357","state":["exists","up"]},{"osd":3,"uuid":"4ae879ca-e6ea-4e19-bdfe-5dcb162ddd30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6826","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6827","nonce":1979857546}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6828","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6829","nonce":1979857546}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6832","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6833","nonce":1979857546}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6830","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6831","nonce":1979857546}]},"public_addr":"172.21.15.3:6827/1979857546","cluster_addr":"172.21.15.3:6829/1979857546","heartbeat_back_addr":"172.21.15.3:6833/1979857546","heartbeat_front_addr":"172.21.15.3:6831/1979857546","state":["exists","up"]},{"osd":4,"uuid":"a3a54374-976e-4d5e-9280-30eb479370ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6801","nonce":3717502855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6803","nonce":3717502855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6807","nonce":3717502855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6805","nonce":3717502855}]},"public_addr":"172.21.15.27:6801/3717502855","cluster_addr":"172.21.15.27:6803/3717502855","heartbeat_back_addr":"172.21.15.27:6807/3717502855","heartbeat_front_addr":"172.21.15.27:6805/3717502855","state":["exists","up"]},{"osd":5,"uuid":"e9bc2fc0-c580-47b3-b37b-91fd8484f319","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6809","nonce":4110125634}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6811","nonce":4110125634}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6815","nonce":4110125634}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6813","nonce":4110125634}]},"public_addr":"172.21.15.27:6809/4110125634","cluster_addr":"172.21.15.27:6811/4110125634","heartbeat_back_addr":"172.21.15.27:6815/4110125634","heartbeat_front_addr":"172.21.15.27:6813/4110125634","state":["exists","up"]},{"osd":6,"uuid":"b8011a9c-5bde-4bd4-8ca5-c2885410df0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6817","nonce":1713832118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6819","nonce":1713832118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6823","nonce":1713832118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6821","nonce":1713832118}]},"public_addr":"172.21.15.27:6817/1713832118","cluster_addr":"172.21.15.27:6819/1713832118","heartbeat_back_addr":"172.21.15.27:6823/1713832118","heartbeat_front_addr":"172.21.15.27:6821/1713832118","state":["exists","up"]},{"osd":7,"uuid":"22d2a92a-4c2d-4a00-a8b8-57d17327fa9a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6825","nonce":3680333329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6827","nonce":3680333329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6831","nonce":3680333329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6829","nonce":3680333329}]},"public_addr":"172.21.15.27:6825/3680333329","cluster_addr":"172.21.15.27:6827/3680333329","heartbeat_back_addr":"172.21.15.27:6831/3680333329","heartbeat_front_addr":"172.21.15.27:6829/3680333329","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:19:23.051172+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:19:53.134218+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:20:21.880355+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:20:49.239590+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:21:10.456027+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:21:34.878758+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:21:58.113324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:22:22.192021+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.3:0/983880676":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/2300598431":"2024-06-02T02:18:24.800609+0000","172.21.15.3:6801/4180696651":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/2960826803":"2024-06-02T02:17:31.096287+0000","172.21.15.3:6801/4010164652":"2024-06-02T02:17:31.096287+0000","172.21.15.3:6800/4010164652":"2024-06-02T02:17:31.096287+0000","172.21.15.3:0/2588496657":"2024-06-02T02:17:31.096287+0000","172.21.15.3:0/2551380483":"2024-06-02T02:17:02.369055+0000","172.21.15.3:6800/4180696651":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/3407438145":"2024-06-02T02:17:31.096287+0000","172.21.15.3:0/378330486":"2024-06-02T02:17:02.369055+0000","172.21.15.3:6800/169228208":"2024-06-02T02:17:02.369055+0000","172.21.15.3:6801/169228208":"2024-06-02T02:17:02.369055+0000","172.21.15.3:0/3124197876":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/3772716787":"2024-06-02T02:17:02.369055+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-01T02:22:42.087 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-06-01T02:22:42.087 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd dump --format=json 2024-06-01T02:22:42.319 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:42.441 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:42 smithi003 ceph-mon[25122]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-06-01T02:22:42.441 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:42 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/838144089' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-01T02:22:42.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:42 smithi027 ceph-mon[27271]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-06-01T02:22:42.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:42 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/838144089' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-01T02:22:44.262 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:22:44.262 INFO:teuthology.orchestra.run.smithi003.stdout:{"epoch":60,"fsid":"e42641dc-1fbc-11ef-bc9b-c7b262605968","created":"2024-06-01T02:16:37.976034+0000","modified":"2024-06-01T02:22:28.191086+0000","last_up_change":"2024-06-01T02:22:24.170381+0000","last_in_change":"2024-06-01T02:22:06.674379+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-01T02:20:25.313034+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fd50417e-a988-4f23-87d1-e45084b46b59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6802","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6803","nonce":2738410731}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6804","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6805","nonce":2738410731}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6808","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6809","nonce":2738410731}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6806","nonce":2738410731},{"type":"v1","addr":"172.21.15.3:6807","nonce":2738410731}]},"public_addr":"172.21.15.3:6803/2738410731","cluster_addr":"172.21.15.3:6805/2738410731","heartbeat_back_addr":"172.21.15.3:6809/2738410731","heartbeat_front_addr":"172.21.15.3:6807/2738410731","state":["exists","up"]},{"osd":1,"uuid":"abe11909-0ec2-46d5-9dcb-2234cfbf5e1c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6810","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6811","nonce":2291814156}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6812","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6813","nonce":2291814156}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6816","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6817","nonce":2291814156}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6814","nonce":2291814156},{"type":"v1","addr":"172.21.15.3:6815","nonce":2291814156}]},"public_addr":"172.21.15.3:6811/2291814156","cluster_addr":"172.21.15.3:6813/2291814156","heartbeat_back_addr":"172.21.15.3:6817/2291814156","heartbeat_front_addr":"172.21.15.3:6815/2291814156","state":["exists","up"]},{"osd":2,"uuid":"02170894-eb8b-446e-b3be-72db8229c089","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6818","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6819","nonce":2180786357}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6820","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6821","nonce":2180786357}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6824","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6825","nonce":2180786357}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6822","nonce":2180786357},{"type":"v1","addr":"172.21.15.3:6823","nonce":2180786357}]},"public_addr":"172.21.15.3:6819/2180786357","cluster_addr":"172.21.15.3:6821/2180786357","heartbeat_back_addr":"172.21.15.3:6825/2180786357","heartbeat_front_addr":"172.21.15.3:6823/2180786357","state":["exists","up"]},{"osd":3,"uuid":"4ae879ca-e6ea-4e19-bdfe-5dcb162ddd30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6826","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6827","nonce":1979857546}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6828","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6829","nonce":1979857546}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6832","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6833","nonce":1979857546}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.3:6830","nonce":1979857546},{"type":"v1","addr":"172.21.15.3:6831","nonce":1979857546}]},"public_addr":"172.21.15.3:6827/1979857546","cluster_addr":"172.21.15.3:6829/1979857546","heartbeat_back_addr":"172.21.15.3:6833/1979857546","heartbeat_front_addr":"172.21.15.3:6831/1979857546","state":["exists","up"]},{"osd":4,"uuid":"a3a54374-976e-4d5e-9280-30eb479370ba","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6801","nonce":3717502855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6803","nonce":3717502855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6807","nonce":3717502855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":3717502855},{"type":"v1","addr":"172.21.15.27:6805","nonce":3717502855}]},"public_addr":"172.21.15.27:6801/3717502855","cluster_addr":"172.21.15.27:6803/3717502855","heartbeat_back_addr":"172.21.15.27:6807/3717502855","heartbeat_front_addr":"172.21.15.27:6805/3717502855","state":["exists","up"]},{"osd":5,"uuid":"e9bc2fc0-c580-47b3-b37b-91fd8484f319","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6809","nonce":4110125634}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6811","nonce":4110125634}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6815","nonce":4110125634}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":4110125634},{"type":"v1","addr":"172.21.15.27:6813","nonce":4110125634}]},"public_addr":"172.21.15.27:6809/4110125634","cluster_addr":"172.21.15.27:6811/4110125634","heartbeat_back_addr":"172.21.15.27:6815/4110125634","heartbeat_front_addr":"172.21.15.27:6813/4110125634","state":["exists","up"]},{"osd":6,"uuid":"b8011a9c-5bde-4bd4-8ca5-c2885410df0b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6817","nonce":1713832118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6819","nonce":1713832118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6823","nonce":1713832118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1713832118},{"type":"v1","addr":"172.21.15.27:6821","nonce":1713832118}]},"public_addr":"172.21.15.27:6817/1713832118","cluster_addr":"172.21.15.27:6819/1713832118","heartbeat_back_addr":"172.21.15.27:6823/1713832118","heartbeat_front_addr":"172.21.15.27:6821/1713832118","state":["exists","up"]},{"osd":7,"uuid":"22d2a92a-4c2d-4a00-a8b8-57d17327fa9a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6825","nonce":3680333329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6827","nonce":3680333329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6831","nonce":3680333329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":3680333329},{"type":"v1","addr":"172.21.15.27:6829","nonce":3680333329}]},"public_addr":"172.21.15.27:6825/3680333329","cluster_addr":"172.21.15.27:6827/3680333329","heartbeat_back_addr":"172.21.15.27:6831/3680333329","heartbeat_front_addr":"172.21.15.27:6829/3680333329","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:19:23.051172+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:19:53.134218+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:20:21.880355+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:20:49.239590+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:21:10.456027+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:21:34.878758+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:21:58.113324+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-01T02:22:22.192021+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.3:0/983880676":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/2300598431":"2024-06-02T02:18:24.800609+0000","172.21.15.3:6801/4180696651":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/2960826803":"2024-06-02T02:17:31.096287+0000","172.21.15.3:6801/4010164652":"2024-06-02T02:17:31.096287+0000","172.21.15.3:6800/4010164652":"2024-06-02T02:17:31.096287+0000","172.21.15.3:0/2588496657":"2024-06-02T02:17:31.096287+0000","172.21.15.3:0/2551380483":"2024-06-02T02:17:02.369055+0000","172.21.15.3:6800/4180696651":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/3407438145":"2024-06-02T02:17:31.096287+0000","172.21.15.3:0/378330486":"2024-06-02T02:17:02.369055+0000","172.21.15.3:6800/169228208":"2024-06-02T02:17:02.369055+0000","172.21.15.3:6801/169228208":"2024-06-02T02:17:02.369055+0000","172.21.15.3:0/3124197876":"2024-06-02T02:18:24.800609+0000","172.21.15.3:0/3772716787":"2024-06-02T02:17:02.369055+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-01T02:22:44.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:44 smithi003 ceph-mon[25122]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:44.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:44 smithi027 ceph-mon[27271]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:45.024 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-06-01T02:22:45.024 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-06-01T02:22:45.025 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-06-01T02:22:45.025 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-06-01T02:22:45.025 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-06-01T02:22:45.026 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-06-01T02:22:45.026 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-06-01T02:22:45.027 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-06-01T02:22:45.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:45 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1799609761' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-01T02:22:45.424 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:45.471 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:45.471 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:45.477 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:45.477 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:45.478 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:45.479 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:45.483 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:45.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:45 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1799609761' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-01T02:22:46.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:46 smithi003 ceph-mon[25122]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:46.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:46 smithi027 ceph-mon[27271]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:48.396 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:48 smithi003 ceph-mon[25122]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:48.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:48 smithi027 ceph-mon[27271]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:50.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:50 smithi003 ceph-mon[25122]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:50.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:50 smithi027 ceph-mon[27271]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:52.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:52 smithi003 ceph-mon[25122]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:52.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:52 smithi027 ceph-mon[27271]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:53.726 INFO:teuthology.orchestra.run.smithi003.stdout:206158430222 2024-06-01T02:22:53.727 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-06-01T02:22:54.148 INFO:teuthology.orchestra.run.smithi003.stdout:137438953495 2024-06-01T02:22:54.148 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-06-01T02:22:54.378 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:54 smithi003 ceph-mon[25122]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:54.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:54 smithi027 ceph-mon[27271]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:56.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:56 smithi003 ceph-mon[25122]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:56.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:56 smithi027 ceph-mon[27271]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:56.704 INFO:teuthology.orchestra.run.smithi003.stdout:171798691858 2024-06-01T02:22:56.705 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-06-01T02:22:58.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:22:58 smithi003 ceph-mon[25122]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:58.515 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:58.515 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:22:58.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:22:58 smithi027 ceph-mon[27271]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:22:59.327 INFO:teuthology.orchestra.run.smithi003.stdout:38654705709 2024-06-01T02:22:59.327 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-06-01T02:22:59.706 INFO:teuthology.orchestra.run.smithi003.stdout:60129542183 2024-06-01T02:22:59.706 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-06-01T02:22:59.945 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:23:00.522 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:00 smithi003 ceph-mon[25122]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:00.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:00 smithi027 ceph-mon[27271]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:01.598 INFO:teuthology.orchestra.run.smithi003.stdout:240518168586 2024-06-01T02:23:01.598 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-06-01T02:23:02.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:02 smithi003 ceph-mon[25122]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:02.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:02 smithi027 ceph-mon[27271]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:03.262 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:23:03.612 INFO:teuthology.orchestra.run.smithi003.stdout:81604378659 2024-06-01T02:23:03.612 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-06-01T02:23:03.805 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:23:04.175 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:23:04.175 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:23:04.509 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:04 smithi003 ceph-mon[25122]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:04.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:04 smithi027 ceph-mon[27271]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:05.441 INFO:teuthology.orchestra.run.smithi003.stdout:38654705710 2024-06-01T02:23:06.489 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:06 smithi003 ceph-mon[25122]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:06.489 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:06 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1315828783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-01T02:23:06.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:06 smithi027 ceph-mon[27271]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:06.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:06 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1315828783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-01T02:23:08.205 INFO:teuthology.orchestra.run.smithi003.stdout:240518168587 2024-06-01T02:23:08.481 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:08 smithi003 ceph-mon[25122]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:08.481 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:08 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1250724049' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-01T02:23:08.481 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:08 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/826982472' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-01T02:23:08.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:08 smithi027 ceph-mon[27271]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:08.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:08 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1250724049' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-01T02:23:08.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:08 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/826982472' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-01T02:23:10.577 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:10 smithi003 ceph-mon[25122]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:10.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:10 smithi027 ceph-mon[27271]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:10.614 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705710 for osd.0 2024-06-01T02:23:10.614 DEBUG:teuthology.parallel:result is None 2024-06-01T02:23:10.615 INFO:teuthology.orchestra.run.smithi003.stdout:115964117022 2024-06-01T02:23:10.615 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-06-01T02:23:11.089 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168586 got 240518168587 for osd.7 2024-06-01T02:23:11.089 DEBUG:teuthology.parallel:result is None 2024-06-01T02:23:11.465 INFO:teuthology.orchestra.run.smithi003.stdout:206158430225 2024-06-01T02:23:12.569 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:12 smithi003 ceph-mon[25122]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:12.569 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:12 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2698016552' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-01T02:23:12.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:12 smithi027 ceph-mon[27271]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:12.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:12 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/2698016552' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-01T02:23:12.979 INFO:teuthology.orchestra.run.smithi003.stdout:171798691861 2024-06-01T02:23:13.170 INFO:teuthology.orchestra.run.smithi003.stdout:60129542185 2024-06-01T02:23:13.533 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:13 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/82033764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-01T02:23:13.533 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:13 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/914241764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-01T02:23:13.533 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:13 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/800528910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-01T02:23:13.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:13 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/82033764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-01T02:23:13.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:13 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/914241764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-01T02:23:13.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:13 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/800528910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-01T02:23:13.911 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542185 for osd.1 2024-06-01T02:23:13.911 DEBUG:teuthology.parallel:result is None 2024-06-01T02:23:13.984 INFO:teuthology.orchestra.run.smithi003.stdout:81604378661 2024-06-01T02:23:14.500 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378659 got 81604378661 for osd.2 2024-06-01T02:23:14.501 DEBUG:teuthology.parallel:result is None 2024-06-01T02:23:14.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:14 smithi003 ceph-mon[25122]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:14.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:14 smithi027 ceph-mon[27271]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:14.646 INFO:teuthology.orchestra.run.smithi003.stdout:137438953497 2024-06-01T02:23:15.233 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953495 got 137438953497 for osd.4 2024-06-01T02:23:15.233 DEBUG:teuthology.parallel:result is None 2024-06-01T02:23:15.674 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430222 got 206158430225 for osd.6 2024-06-01T02:23:15.674 DEBUG:teuthology.parallel:result is None 2024-06-01T02:23:16.078 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691858 got 171798691861 for osd.5 2024-06-01T02:23:16.078 DEBUG:teuthology.parallel:result is None 2024-06-01T02:23:16.116 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:23:16.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:16 smithi003 ceph-mon[25122]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:16.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:16 smithi027 ceph-mon[27271]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:17.763 INFO:teuthology.orchestra.run.smithi003.stdout:115964117024 2024-06-01T02:23:18.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:18 smithi003 ceph-mon[25122]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:18.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:18 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1517952014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-01T02:23:18.367 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117022 got 115964117024 for osd.3 2024-06-01T02:23:18.367 DEBUG:teuthology.parallel:result is None 2024-06-01T02:23:18.367 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-06-01T02:23:18.367 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph pg dump --format=json 2024-06-01T02:23:18.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:18 smithi027 ceph-mon[27271]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:18.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:18 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1517952014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-01T02:23:18.638 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:23:20.335 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:23:20.337 INFO:teuthology.orchestra.run.smithi003.stderr:dumped all 2024-06-01T02:23:20.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:20 smithi003 ceph-mon[25122]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:20.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:20 smithi027 ceph-mon[27271]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:20.976 INFO:teuthology.orchestra.run.smithi003.stdout:{"pg_ready":true,"pg_map":{"version":195,"stamp":"2024-06-01T02:23:18.862286+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48940,"kb_used_data":6380,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912276,"statfs":{"total":767960285184,"available":767910170624,"internally_reserved":0,"allocated":6533120,"data_stored":3572009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002453"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-06-01T02:22:28.197609+0000","last_change":"2024-06-01T02:22:28.197609+0000","last_active":"2024-06-01T02:22:28.197609+0000","last_peered":"2024-06-01T02:22:28.197609+0000","last_clean":"2024-06-01T02:22:28.197609+0000","last_became_active":"2024-06-01T02:22:28.197014+0000","last_became_peered":"2024-06-01T02:22:28.197014+0000","last_unstale":"2024-06-01T02:22:28.197609+0000","last_undegraded":"2024-06-01T02:22:28.197609+0000","last_fullsized":"2024-06-01T02:22:28.197609+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-06-01T02:20:25.829913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-06-01T02:20:25.829913+0000","last_clean_scrub_stamp":"2024-06-01T02:20:25.829913+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-06-02T08:25:51.614940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168589,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61099999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59699999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38300000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64700000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.505}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6124,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739028,"statfs":{"total":95995035648,"available":95988764672,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.9390000000000001,"5min":5.9390000000000001,"15min":5.9390000000000001},"last":0.495}]},{"osd":1,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.58699999999999997}]},{"osd":2,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.9809999999999999,"5min":5.9809999999999999,"15min":5.9809999999999999},"last":0.47599999999999998}]},{"osd":3,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.57499999999999996}]},{"osd":4,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.66300000000000003}]},{"osd":5,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.62},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.68300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64800000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6368,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738784,"statfs":{"total":95995035648,"available":95988514816,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:22:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.36499999999999999,"15min":0.34599999999999997},"min":{"1min":0.29199999999999998,"5min":0.218,"15min":0.218},"max":{"1min":0.59399999999999997,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.378},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.375,"15min":0.34699999999999998},"min":{"1min":0.27200000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.52900000000000003,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Jun 1 02:22:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38300000000000001,"15min":0.38100000000000001},"min":{"1min":0.28499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.55900000000000005,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.315},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.41799999999999998,"15min":0.40799999999999997},"min":{"1min":0.34399999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Jun 1 02:22:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.37},{"interface":"front","average":{"1min":0.437,"5min":0.44900000000000001,"15min":0.45100000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.622,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.60899999999999999,"15min":0.61499999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78700000000000003,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.64500000000000002,"15min":0.65000000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80200000000000005,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.75700000000000001}]},{"osd":5,"last update":"Sat Jun 1 02:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.77200000000000002}]},{"osd":6,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":10.055,"5min":10.055,"15min":10.055},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.80800000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72899999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 1 02:22:59 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.37,"15min":0.35699999999999998},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":0.57099999999999995,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.438,"5min":0.378,"15min":0.35499999999999998},"min":{"1min":0.33000000000000002,"5min":0.216,"15min":0.216},"max":{"1min":0.623,"5min":0.626,"15min":0.626},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Jun 1 02:22:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.40699999999999997,"15min":0.40000000000000002},"min":{"1min":0.33700000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.434,"15min":0.42899999999999999},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.56299999999999994,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.34899999999999998}]},{"osd":3,"last update":"Sat Jun 1 02:22:55 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.501,"15min":0.503},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.55800000000000005,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Jun 1 02:22:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.79600000000000004}]},{"osd":5,"last update":"Sat Jun 1 02:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.70999999999999996}]},{"osd":6,"last update":"Sat Jun 1 02:23:05 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.72799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78600000000000003}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:22:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.36399999999999999,"15min":0.35299999999999998},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.38300000000000001,"15min":0.373},"min":{"1min":0.29999999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.40100000000000002}]},{"osd":1,"last update":"Sat Jun 1 02:22:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.39400000000000002,"15min":0.38},"min":{"1min":0.32000000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.38300000000000001,"15min":0.374},"min":{"1min":0.308,"5min":0.254,"15min":0.254},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.34999999999999998}]},{"osd":3,"last update":"Sat Jun 1 02:22:56 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.499,"15min":0.504},"min":{"1min":0.34499999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.48499999999999999,"15min":0.49099999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.379}]},{"osd":4,"last update":"Sat Jun 1 02:22:13 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.70999999999999996}]},{"osd":5,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.66600000000000004}]},{"osd":6,"last update":"Sat Jun 1 02:23:00 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.64500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60599999999999998}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40799999999999997,"15min":0.40600000000000003},"min":{"1min":0.309,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.32800000000000001}]},{"osd":1,"last update":"Sat Jun 1 02:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.432,"15min":0.433},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.42399999999999999,"15min":0.41999999999999998},"min":{"1min":0.33700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.245}]},{"osd":2,"last update":"Sat Jun 1 02:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.59999999999999998,"5min":0.63,"15min":0.63},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.44400000000000001,"15min":0.44800000000000001},"min":{"1min":0.30399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.374}]},{"osd":4,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.42599999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.56699999999999995,"15min":0.55400000000000005},"min":{"1min":0.47699999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.63500000000000001}]},{"osd":5,"last update":"Sat Jun 1 02:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Jun 1 02:23:01 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.60699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65500000000000003}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.54800000000000004,"15min":0.55500000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.57099999999999995,"15min":0.57199999999999995},"min":{"1min":0.41999999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.79200000000000004}]},{"osd":1,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71599999999999997,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56699999999999995,"15min":0.57399999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.73199999999999998,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.65500000000000003}]},{"osd":2,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.59199999999999997,"15min":0.59999999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.78000000000000003,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.39200000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.82199999999999995}]},{"osd":3,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56000000000000005,"15min":0.55600000000000005},"min":{"1min":0.42999999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.78200000000000003,"15min":0.81799999999999995},"min":{"1min":0.39100000000000001,"5min":0.378,"15min":0.378},"max":{"1min":0.76700000000000002,"5min":5.5579999999999998,"15min":5.5579999999999998},"last":0.75900000000000001}]},{"osd":5,"last update":"Sat Jun 1 02:22:39 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.625}]},{"osd":6,"last update":"Sat Jun 1 02:23:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.58399999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68500000000000005}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.40799999999999997}]},{"osd":1,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.53000000000000003}]},{"osd":3,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.627},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.48899999999999999}]},{"osd":4,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.64800000000000002}]},{"osd":6,"last update":"Sat Jun 1 02:23:03 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.66800000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-06-01T02:23:20.977 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph pg dump --format=json 2024-06-01T02:23:21.201 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:23:21.562 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:21 smithi003 ceph-mon[25122]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:21.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:21 smithi027 ceph-mon[27271]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:22.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:22 smithi003 ceph-mon[25122]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:22.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:22 smithi027 ceph-mon[27271]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:22.909 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:23:22.910 INFO:teuthology.orchestra.run.smithi003.stderr:dumped all 2024-06-01T02:23:23.532 INFO:teuthology.orchestra.run.smithi003.stdout:{"pg_ready":true,"pg_map":{"version":197,"stamp":"2024-06-01T02:23:22.863184+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48940,"kb_used_data":6380,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912276,"statfs":{"total":767960285184,"available":767910170624,"internally_reserved":0,"allocated":6533120,"data_stored":3572009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002485"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-06-01T02:22:28.197609+0000","last_change":"2024-06-01T02:22:28.197609+0000","last_active":"2024-06-01T02:22:28.197609+0000","last_peered":"2024-06-01T02:22:28.197609+0000","last_clean":"2024-06-01T02:22:28.197609+0000","last_became_active":"2024-06-01T02:22:28.197014+0000","last_became_peered":"2024-06-01T02:22:28.197014+0000","last_unstale":"2024-06-01T02:22:28.197609+0000","last_undegraded":"2024-06-01T02:22:28.197609+0000","last_fullsized":"2024-06-01T02:22:28.197609+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-06-01T02:20:25.829913+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-06-01T02:20:25.829913+0000","last_clean_scrub_stamp":"2024-06-01T02:20:25.829913+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-06-02T08:25:51.614940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68200000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53600000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65000000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63200000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77500000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72199999999999998}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6124,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739028,"statfs":{"total":95995035648,"available":95988764672,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.9390000000000001,"5min":5.9390000000000001,"15min":5.9390000000000001},"last":0.74199999999999999}]},{"osd":1,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.59399999999999997}]},{"osd":2,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.9809999999999999,"5min":5.9809999999999999,"15min":5.9809999999999999},"last":0.624}]},{"osd":3,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.65100000000000002}]},{"osd":4,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.57999999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69499999999999995}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6368,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738784,"statfs":{"total":95995035648,"available":95988514816,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:22:58 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.36499999999999999,"15min":0.34599999999999997},"min":{"1min":0.29199999999999998,"5min":0.218,"15min":0.218},"max":{"1min":0.59399999999999997,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.375,"15min":0.34699999999999998},"min":{"1min":0.27200000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.52900000000000003,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Jun 1 02:22:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38300000000000001,"15min":0.38100000000000001},"min":{"1min":0.28499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.55900000000000005,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.41799999999999998,"15min":0.40799999999999997},"min":{"1min":0.34399999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46600000000000003}]},{"osd":3,"last update":"Sat Jun 1 02:22:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.504},{"interface":"front","average":{"1min":0.437,"5min":0.44900000000000001,"15min":0.45100000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.622,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.60899999999999999,"15min":0.61499999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78700000000000003,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.64500000000000002,"15min":0.65000000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80200000000000005,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.73699999999999999}]},{"osd":5,"last update":"Sat Jun 1 02:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.75800000000000001}]},{"osd":6,"last update":"Sat Jun 1 02:23:02 2024","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":10.055,"5min":10.055,"15min":10.055},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.60299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71899999999999997}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 1 02:22:59 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.37,"15min":0.35699999999999998},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":0.57099999999999995,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.438,"5min":0.378,"15min":0.35499999999999998},"min":{"1min":0.33000000000000002,"5min":0.216,"15min":0.216},"max":{"1min":0.623,"5min":0.626,"15min":0.626},"last":0.44900000000000001}]},{"osd":2,"last update":"Sat Jun 1 02:22:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.40699999999999997,"15min":0.40000000000000002},"min":{"1min":0.33700000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.434,"15min":0.42899999999999999},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.56299999999999994,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.34799999999999998}]},{"osd":3,"last update":"Sat Jun 1 02:22:55 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.501,"15min":0.503},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.55800000000000005,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Jun 1 02:22:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.74099999999999999}]},{"osd":5,"last update":"Sat Jun 1 02:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.70999999999999996}]},{"osd":6,"last update":"Sat Jun 1 02:23:05 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.67500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:22:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.36399999999999999,"15min":0.35299999999999998},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.38300000000000001,"15min":0.373},"min":{"1min":0.29999999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Jun 1 02:22:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.39400000000000002,"15min":0.38},"min":{"1min":0.32000000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.378},{"interface":"front","average":{"1min":0.437,"5min":0.38300000000000001,"15min":0.374},"min":{"1min":0.308,"5min":0.254,"15min":0.254},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.43099999999999999}]},{"osd":3,"last update":"Sat Jun 1 02:22:56 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.499,"15min":0.504},"min":{"1min":0.34499999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.48499999999999999,"15min":0.49099999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.503}]},{"osd":4,"last update":"Sat Jun 1 02:23:16 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.61399999999999999,"15min":0.60599999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.63,"15min":0.628},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.82899999999999996}]},{"osd":5,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.80000000000000004}]},{"osd":6,"last update":"Sat Jun 1 02:23:00 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.71099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78100000000000003}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40799999999999997,"15min":0.40600000000000003},"min":{"1min":0.309,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.314}]},{"osd":1,"last update":"Sat Jun 1 02:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.432,"15min":0.433},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.42399999999999999,"15min":0.41999999999999998},"min":{"1min":0.33700000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Jun 1 02:22:53 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.59999999999999998,"5min":0.63,"15min":0.63},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.44400000000000001,"15min":0.44800000000000001},"min":{"1min":0.30399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.42599999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.56699999999999995,"15min":0.55400000000000005},"min":{"1min":0.47699999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.623}]},{"osd":5,"last update":"Sat Jun 1 02:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Jun 1 02:23:01 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.68700000000000006}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60599999999999998}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.54800000000000004,"15min":0.55500000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.57099999999999995,"15min":0.57199999999999995},"min":{"1min":0.41999999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71599999999999997,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56699999999999995,"15min":0.57399999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.73199999999999998,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.38700000000000001}]},{"osd":2,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.59199999999999997,"15min":0.59999999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.78000000000000003,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.39200000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.57199999999999995}]},{"osd":3,"last update":"Sat Jun 1 02:23:15 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56000000000000005,"15min":0.55600000000000005},"min":{"1min":0.42999999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.78200000000000003,"15min":0.81799999999999995},"min":{"1min":0.39100000000000001,"5min":0.378,"15min":0.378},"max":{"1min":0.76700000000000002,"5min":5.5579999999999998,"15min":5.5579999999999998},"last":0.55200000000000005}]},{"osd":5,"last update":"Sat Jun 1 02:22:39 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.436},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.68500000000000005}]},{"osd":6,"last update":"Sat Jun 1 02:23:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.71199999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63700000000000001}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.60899999999999999}]},{"osd":1,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.54700000000000004}]},{"osd":2,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.437}]},{"osd":3,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.52600000000000002}]},{"osd":4,"last update":"Sat Jun 1 02:22:40 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.502}]},{"osd":6,"last update":"Sat Jun 1 02:23:03 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.624}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-06-01T02:23:23.534 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-06-01T02:23:23.534 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-06-01T02:23:23.534 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-06-01T02:23:23.534 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph health --format=json 2024-06-01T02:23:23.761 INFO:teuthology.orchestra.run.smithi003.stderr:Inferring config /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/mon.smithi003/config 2024-06-01T02:23:24.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:24 smithi003 ceph-mon[25122]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:24.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:24 smithi003 ceph-mon[25122]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:24.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:24 smithi027 ceph-mon[27271]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:24.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:24 smithi027 ceph-mon[27271]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:25.472 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:23:25.472 INFO:teuthology.orchestra.run.smithi003.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-06-01T02:23:25.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:23:25.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:23:25.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:23:25.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:23:26.070 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-06-01T02:23:26.070 INFO:tasks.cephadm:Setup complete, yielding 2024-06-01T02:23:26.070 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-01T02:23:26.079 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:23:26.080 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch status' 2024-06-01T02:23:26.523 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:26 smithi003 ceph-mon[25122]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:26.523 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:26 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/4119140548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-01T02:23:26.523 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:23:26.523 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:26.523 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:23:26.524 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:26.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:26 smithi027 ceph-mon[27271]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:26.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:26 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/4119140548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-01T02:23:26.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:23:26.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:23:26.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:28.135 INFO:teuthology.orchestra.run.smithi003.stdout:Backend: cephadm 2024-06-01T02:23:28.135 INFO:teuthology.orchestra.run.smithi003.stdout:Available: Yes 2024-06-01T02:23:28.135 INFO:teuthology.orchestra.run.smithi003.stdout:Paused: No 2024-06-01T02:23:28.434 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:28 smithi003 ceph-mon[25122]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:28.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:28 smithi027 ceph-mon[27271]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:28.711 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch ps' 2024-06-01T02:23:29.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:29 smithi003 ceph-mon[25122]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:29.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:29 smithi027 ceph-mon[27271]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:30.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:30 smithi003 ceph-mon[25122]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:30.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:30 smithi027 ceph-mon[27271]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:30.711 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:23:30.711 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (4m) 2m ago 5m 17.6M - ba2b418f427c ceda73487cd9 2024-06-01T02:23:30.711 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (5m) 2m ago 5m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:23:30.711 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (4m) 67s ago 4m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:23:30.711 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (3m) 2m ago 5m 43.2M - 8.3.5 dad864ee21e9 32cd8820aa5e 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:9283 running (6m) 2m ago 6m 450M - 17.2.0 e1d6a67b021e ac6fbe8a9d2d 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283 running (4m) 67s ago 4m 412M - 17.2.0 e1d6a67b021e e778130b1363 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (6m) 2m ago 6m 41.0M 2048M 17.2.0 e1d6a67b021e 108dda5fe7cc 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (4m) 67s ago 4m 42.4M 2048M 17.2.0 e1d6a67b021e edc537c54a3a 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (5m) 2m ago 5m 18.2M - 1dbe0e931976 6338b03aebaf 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (4m) 67s ago 4m 16.9M - 1dbe0e931976 3ee520f9eebb 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (4m) 2m ago 4m 33.9M 3217M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (3m) 2m ago 3m 35.6M 3217M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (3m) 2m ago 3m 31.2M 3217M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (2m) 2m ago 2m 24.7M 3217M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (2m) 67s ago 2m 33.6M 3984M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (2m) 67s ago 119s 32.2M 3984M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:23:30.712 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (97s) 67s ago 96s 30.2M 3984M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:23:30.713 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (73s) 67s ago 72s 13.4M 3984M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:23:30.713 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (4m) 2m ago 4m 53.4M - 514e6a882f6e 70b2c7c41ae9 2024-06-01T02:23:31.391 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch ls' 2024-06-01T02:23:32.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:32 smithi003 ceph-mon[25122]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:32.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:32 smithi003 ceph-mon[25122]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:32.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:32 smithi027 ceph-mon[27271]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:32.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:32 smithi027 ceph-mon[27271]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:33.398 INFO:teuthology.orchestra.run.smithi003.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-06-01T02:23:33.398 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-06-01T02:23:33.398 INFO:teuthology.orchestra.run.smithi003.stdout:crash 2/2 2m ago 6m * 2024-06-01T02:23:33.398 INFO:teuthology.orchestra.run.smithi003.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-06-01T02:23:33.398 INFO:teuthology.orchestra.run.smithi003.stdout:mgr 2/2 2m ago 6m count:2 2024-06-01T02:23:33.399 INFO:teuthology.orchestra.run.smithi003.stdout:mon 2/2 2m ago 5m smithi003:172.21.15.3=smithi003;smithi027:172.21.15.27=smithi027;count:2 2024-06-01T02:23:33.399 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-06-01T02:23:33.399 INFO:teuthology.orchestra.run.smithi003.stdout:osd 8 2m ago - 2024-06-01T02:23:33.399 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-06-01T02:23:33.981 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch host ls' 2024-06-01T02:23:34.550 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:34 smithi003 ceph-mon[25122]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:34.551 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:34 smithi003 ceph-mon[25122]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:34.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:34 smithi027 ceph-mon[27271]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:34.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:34 smithi027 ceph-mon[27271]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:35.987 INFO:teuthology.orchestra.run.smithi003.stdout:HOST ADDR LABELS STATUS 2024-06-01T02:23:35.987 INFO:teuthology.orchestra.run.smithi003.stdout:smithi003 172.21.15.3 2024-06-01T02:23:35.987 INFO:teuthology.orchestra.run.smithi003.stdout:smithi027 172.21.15.27 2024-06-01T02:23:35.987 INFO:teuthology.orchestra.run.smithi003.stdout:2 hosts in cluster 2024-06-01T02:23:36.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:36 smithi003 ceph-mon[25122]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:36.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:36 smithi027 ceph-mon[27271]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:36.623 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch device ls' 2024-06-01T02:23:37.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:37 smithi003 ceph-mon[25122]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:37.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:37 smithi027 ceph-mon[27271]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:38.505 INFO:teuthology.orchestra.run.smithi003.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-06-01T02:23:38.505 INFO:teuthology.orchestra.run.smithi003.stdout:smithi003 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5330009Z400BGN 400G LVM detected, locked 2024-06-01T02:23:38.505 INFO:teuthology.orchestra.run.smithi003.stdout:smithi027 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5331001C400BGN 400G LVM detected, locked 2024-06-01T02:23:38.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:38 smithi003 ceph-mon[25122]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:38.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:38 smithi027 ceph-mon[27271]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:39.167 INFO:teuthology.run_tasks:Running task vip.exec... 2024-06-01T02:23:39.179 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:23:39.180 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-06-01T02:23:39.207 INFO:teuthology.orchestra.run.smithi003.stderr:+ systemctl stop nfs-server 2024-06-01T02:23:39.216 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi027.front.sepia.ceph.com 2024-06-01T02:23:39.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-06-01T02:23:39.247 INFO:teuthology.orchestra.run.smithi027.stderr:+ systemctl stop nfs-server 2024-06-01T02:23:39.256 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-01T02:23:39.265 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:23:39.265 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-06-01T02:23:39.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:39 smithi003 ceph-mon[25122]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:39.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:39 smithi027 ceph-mon[27271]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:40.594 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:40 smithi003 ceph-mon[25122]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:40.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:40 smithi027 ceph-mon[27271]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:42.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:42 smithi027 ceph-mon[27271]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:42.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:42 smithi027 ceph-mon[27271]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:42.399 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:42 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-06-01T02:23:42.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:42 smithi003 ceph-mon[25122]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:42.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:42 smithi003 ceph-mon[25122]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:23:42.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:42 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-06-01T02:23:43.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:42 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003[25118]: 2024-06-01T02:23:42.708+0000 7fd5d501a700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: osdmap e61: 8 total, 8 up, 8 in 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: osdmap e62: 8 total, 8 up, 8 in 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: osdmap e63: 8 total, 8 up, 8 in 2024-06-01T02:23:43.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: fsmap foofs:0 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi003.kexxec", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi003.kexxec", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-06-01T02:23:43.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:43 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:43.364 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-06-01T02:23:43.375 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-06-01T02:23:43.375 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-01T02:23:43.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-06-01T02:23:43.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: osdmap e61: 8 total, 8 up, 8 in 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: osdmap e62: 8 total, 8 up, 8 in 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: osdmap e63: 8 total, 8 up, 8 in 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: fsmap foofs:0 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:43.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:23:43.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:43.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:23:43.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:43.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:43.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi003.kexxec", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:23:43.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi003.kexxec", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-06-01T02:23:43.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:43 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:44.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:44 smithi003 ceph-mon[25122]: Saving service mds.foofs spec with placement count:2 2024-06-01T02:23:44.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:44 smithi003 ceph-mon[25122]: Deploying daemon mds.foofs.smithi003.kexxec on smithi003 2024-06-01T02:23:44.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:44 smithi003 ceph-mon[25122]: pgmap v210: 65 pgs: 9 creating+peering, 1 active+clean, 55 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:44.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:44 smithi003 ceph-mon[25122]: osdmap e64: 8 total, 8 up, 8 in 2024-06-01T02:23:44.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:44 smithi027 ceph-mon[27271]: Saving service mds.foofs spec with placement count:2 2024-06-01T02:23:44.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:44 smithi027 ceph-mon[27271]: Deploying daemon mds.foofs.smithi003.kexxec on smithi003 2024-06-01T02:23:44.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:44 smithi027 ceph-mon[27271]: pgmap v210: 65 pgs: 9 creating+peering, 1 active+clean, 55 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:44.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:44 smithi027 ceph-mon[27271]: osdmap e64: 8 total, 8 up, 8 in 2024-06-01T02:23:45.907 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:45 smithi003 ceph-mon[25122]: osdmap e65: 8 total, 8 up, 8 in 2024-06-01T02:23:45.908 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:45 smithi003 ceph-mon[25122]: pgmap v213: 65 pgs: 9 creating+peering, 5 active+clean, 51 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:46.096 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:23:46.096 INFO:teuthology.orchestra.run.smithi003.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-01T02:17:20.272253Z", "last_refresh": "2024-06-01T02:20:58.954939Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-01T02:17:15.344910Z", "last_refresh": "2024-06-01T02:20:58.955009Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-06-01T02:17:17.933675Z", "last_refresh": "2024-06-01T02:20:58.955077Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-06-01T02:23:42.733174Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-01T02:23:42.727611Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-01T02:17:13.392361Z", "last_refresh": "2024-06-01T02:20:58.954845Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi003:172.21.15.3=smithi003", "smithi027:172.21.15.27=smithi027"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-01T02:18:03.189384Z", "last_refresh": "2024-06-01T02:20:58.954696Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-01T02:17:19.190358Z", "last_refresh": "2024-06-01T02:20:58.955143Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-06-01T02:20:58.955272Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-01T02:17:16.680096Z", "last_refresh": "2024-06-01T02:20:58.955208Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-01T02:23:46.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:45 smithi027 ceph-mon[27271]: osdmap e65: 8 total, 8 up, 8 in 2024-06-01T02:23:46.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:45 smithi027 ceph-mon[27271]: pgmap v213: 65 pgs: 9 creating+peering, 5 active+clean, 51 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:47.088 INFO:tasks.cephadm:mds.foofs has 0/2 2024-06-01T02:23:47.847 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi027.xaykap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi027.xaykap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: Deploying daemon mds.foofs.smithi027.xaykap on smithi027 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:boot 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: daemon mds.foofs.smithi003.kexxec assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: Cluster is now healthy 2024-06-01T02:23:47.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: fsmap foofs:0 1 up:standby 2024-06-01T02:23:47.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:creating} 2024-06-01T02:23:47.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:23:47.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:47 smithi027 ceph-mon[27271]: daemon mds.foofs.smithi003.kexxec is now active in filesystem foofs as rank 0 2024-06-01T02:23:48.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi027.xaykap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi027.xaykap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: Deploying daemon mds.foofs.smithi027.xaykap on smithi027 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:boot 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: daemon mds.foofs.smithi003.kexxec assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: Cluster is now healthy 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: fsmap foofs:0 1 up:standby 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:creating} 2024-06-01T02:23:48.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:23:48.083 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:47 smithi003 ceph-mon[25122]: daemon mds.foofs.smithi003.kexxec is now active in filesystem foofs as rank 0 2024-06-01T02:23:48.090 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-01T02:23:48.735 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:48 smithi003 ceph-mon[25122]: pgmap v214: 65 pgs: 9 creating+peering, 41 active+clean, 15 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:48.735 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:48 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:23:48.735 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:48 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 2024-06-01T02:23:48.761 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:48 smithi027 ceph-mon[27271]: pgmap v214: 65 pgs: 9 creating+peering, 41 active+clean, 15 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:23:48.761 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:48 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:23:48.761 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:48 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 2024-06-01T02:23:50.137 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:23:50.137 INFO:teuthology.orchestra.run.smithi003.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-01T02:17:20.272253Z", "last_refresh": "2024-06-01T02:20:58.954939Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-01T02:17:15.344910Z", "last_refresh": "2024-06-01T02:20:58.955009Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-06-01T02:17:17.933675Z", "last_refresh": "2024-06-01T02:20:58.955077Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-06-01T02:23:42.733174Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-01T02:23:42.727611Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-01T02:17:13.392361Z", "last_refresh": "2024-06-01T02:20:58.954845Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi003:172.21.15.3=smithi003", "smithi027:172.21.15.27=smithi027"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-01T02:18:03.189384Z", "last_refresh": "2024-06-01T02:20:58.954696Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-01T02:17:19.190358Z", "last_refresh": "2024-06-01T02:20:58.955143Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-06-01T02:20:58.955272Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-01T02:17:16.680096Z", "last_refresh": "2024-06-01T02:20:58.955208Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-01T02:23:50.461 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: pgmap v215: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 166 B/s wr, 0 op/s 2024-06-01T02:23:50.462 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:50.462 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:23:50.462 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:50.462 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:23:50.462 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] up:boot 2024-06-01T02:23:50.462 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:23:50.462 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:23:50.462 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:23:50.462 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:50 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:50.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: pgmap v215: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 166 B/s wr, 0 op/s 2024-06-01T02:23:50.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:50.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:23:50.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:50.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:23:50.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] up:boot 2024-06-01T02:23:50.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:23:50.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:23:50.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:23:50.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:50 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:51.267 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:51 smithi003 ceph-mon[25122]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:51.292 INFO:tasks.cephadm:mds.foofs has 0/2 2024-06-01T02:23:51.424 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:51 smithi027 ceph-mon[27271]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:52.293 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-01T02:23:52.522 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:52 smithi003 ceph-mon[25122]: pgmap v216: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-06-01T02:23:52.522 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:52.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:52 smithi027 ceph-mon[27271]: pgmap v216: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-06-01T02:23:52.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:54.268 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:54 smithi003 ceph-mon[25122]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-06-01T02:23:54.573 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:23:54.574 INFO:teuthology.orchestra.run.smithi003.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-01T02:17:20.272253Z", "last_refresh": "2024-06-01T02:20:58.954939Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-01T02:17:15.344910Z", "last_refresh": "2024-06-01T02:20:58.955009Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-06-01T02:17:17.933675Z", "last_refresh": "2024-06-01T02:20:58.955077Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-06-01T02:23:42.733174Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-01T02:23:42.727611Z", "last_refresh": "2024-06-01T02:23:51.190685Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-01T02:17:13.392361Z", "last_refresh": "2024-06-01T02:20:58.954845Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi003:172.21.15.3=smithi003", "smithi027:172.21.15.27=smithi027"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-01T02:18:03.189384Z", "last_refresh": "2024-06-01T02:20:58.954696Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-01T02:17:19.190358Z", "last_refresh": "2024-06-01T02:20:58.955143Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-06-01T02:20:58.955272Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-01T02:17:16.680096Z", "last_refresh": "2024-06-01T02:20:58.955208Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-01T02:23:54.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:54 smithi027 ceph-mon[27271]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-06-01T02:23:55.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:55 smithi003 ceph-mon[25122]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:55.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:55.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:55 smithi027 ceph-mon[27271]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:23:55.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:56.070 INFO:tasks.cephadm:mds.foofs has 1/2 2024-06-01T02:23:56.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:56 smithi003 ceph-mon[25122]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-06-01T02:23:56.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:56 smithi027 ceph-mon[27271]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-06-01T02:23:57.072 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-01T02:23:57.398 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:57 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:57.398 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:57 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:57.398 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:57 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:23:57.398 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:57 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:57.398 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:57 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:23:57.398 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:57 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:57.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:57 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:57.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:57 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:57.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:57 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:23:57.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:57 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:23:57.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:57 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:23:57.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:57 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:23:58.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:23:58 smithi003 ceph-mon[25122]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-06-01T02:23:58.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:23:58 smithi027 ceph-mon[27271]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-06-01T02:23:59.125 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:23:59.125 INFO:teuthology.orchestra.run.smithi003.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-01T02:17:20.272253Z", "last_refresh": "2024-06-01T02:23:56.285775Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-01T02:17:15.344910Z", "last_refresh": "2024-06-01T02:23:51.190032Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-06-01T02:17:17.933675Z", "last_refresh": "2024-06-01T02:23:56.285934Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-06-01T02:23:42.733174Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-01T02:23:42.727611Z", "last_refresh": "2024-06-01T02:23:51.190685Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-01T02:17:13.392361Z", "last_refresh": "2024-06-01T02:23:51.190188Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi003:172.21.15.3=smithi003", "smithi027:172.21.15.27=smithi027"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-01T02:18:03.189384Z", "last_refresh": "2024-06-01T02:23:51.190268Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-01T02:17:19.190358Z", "last_refresh": "2024-06-01T02:23:51.190339Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-06-01T02:23:51.190410Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-01T02:17:16.680096Z", "last_refresh": "2024-06-01T02:23:56.286069Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-01T02:23:59.735 INFO:tasks.cephadm:mds.foofs has 2/2 2024-06-01T02:23:59.735 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-01T02:23:59.745 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:23:59.745 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-06-01T02:24:00.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:00 smithi003 ceph-mon[25122]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-06-01T02:24:00.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:00 smithi003 ceph-mon[25122]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:24:00.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:00 smithi027 ceph-mon[27271]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-06-01T02:24:00.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:00 smithi027 ceph-mon[27271]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:24:02.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:02 smithi003 ceph-mon[25122]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-06-01T02:24:02.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-06-01T02:24:02.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:02 smithi027 ceph-mon[27271]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-06-01T02:24:02.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-06-01T02:24:03.346 INFO:teuthology.orchestra.run.smithi003.stdout:NFS Cluster Created Successfully 2024-06-01T02:24:03.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:03 smithi003 ceph-mon[25122]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:03.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:03 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-06-01T02:24:03.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:03 smithi003 ceph-mon[25122]: osdmap e66: 8 total, 8 up, 8 in 2024-06-01T02:24:03.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:03 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-06-01T02:24:03.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:03 smithi027 ceph-mon[27271]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:03.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:03 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-06-01T02:24:03.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:03 smithi027 ceph-mon[27271]: osdmap e66: 8 total, 8 up, 8 in 2024-06-01T02:24:03.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:03 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-06-01T02:24:04.132 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-06-01T02:24:04.403 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: pgmap v223: 97 pgs: 9 creating+peering, 23 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-06-01T02:24:04.403 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: osdmap e67: 8 total, 8 up, 8 in 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: Saving service nfs.foo spec with placement count:2 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: Creating key for client.nfs.foo.0.0.smithi003.dmybfb 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-06-01T02:24:04.404 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-01T02:24:04.405 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-01T02:24:04.405 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:04.405 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-01T02:24:04.405 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-01T02:24:04.405 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-01T02:24:04.405 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-01T02:24:04.405 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:04 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:04.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: pgmap v223: 97 pgs: 9 creating+peering, 23 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-06-01T02:24:04.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: osdmap e67: 8 total, 8 up, 8 in 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: Saving service nfs.foo spec with placement count:2 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: Creating key for client.nfs.foo.0.0.smithi003.dmybfb 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-06-01T02:24:04.598 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-06-01T02:24:04.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-01T02:24:04.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-01T02:24:04.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:04.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-01T02:24:04.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-01T02:24:04.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-01T02:24:04.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-01T02:24:04.599 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:04 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:05.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:05 smithi003 ceph-mon[25122]: Rados config object exists: conf-nfs.foo 2024-06-01T02:24:05.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:05 smithi003 ceph-mon[25122]: Creating key for client.nfs.foo.0.0.smithi003.dmybfb-rgw 2024-06-01T02:24:05.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:05 smithi003 ceph-mon[25122]: Deploying daemon nfs.foo.0.0.smithi003.dmybfb on smithi003 2024-06-01T02:24:05.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:05 smithi003 ceph-mon[25122]: osdmap e68: 8 total, 8 up, 8 in 2024-06-01T02:24:05.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:05 smithi027 ceph-mon[27271]: Rados config object exists: conf-nfs.foo 2024-06-01T02:24:05.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:05 smithi027 ceph-mon[27271]: Creating key for client.nfs.foo.0.0.smithi003.dmybfb-rgw 2024-06-01T02:24:05.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:05 smithi027 ceph-mon[27271]: Deploying daemon nfs.foo.0.0.smithi003.dmybfb on smithi003 2024-06-01T02:24:05.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:05 smithi027 ceph-mon[27271]: osdmap e68: 8 total, 8 up, 8 in 2024-06-01T02:24:06.514 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:06 smithi003 ceph-mon[25122]: pgmap v226: 97 pgs: 9 creating+peering, 22 unknown, 66 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:24:06.514 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:06 smithi003 ceph-mon[25122]: mgrmap e20: smithi003.njsqxz(active, since 5m), standbys: smithi027.tujskj 2024-06-01T02:24:06.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:06 smithi027 ceph-mon[27271]: pgmap v226: 97 pgs: 9 creating+peering, 22 unknown, 66 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:24:06.597 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:06 smithi027 ceph-mon[27271]: mgrmap e20: smithi003.njsqxz(active, since 5m), standbys: smithi027.tujskj 2024-06-01T02:24:07.695 INFO:teuthology.orchestra.run.smithi003.stderr:Invalid command: Unexpected argument '--clusterid' 2024-06-01T02:24:07.696 INFO:teuthology.orchestra.run.smithi003.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-06-01T02:24:07.696 INFO:teuthology.orchestra.run.smithi003.stderr:Error EINVAL: invalid command 2024-06-01T02:24:08.046 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:24:08.046 INFO:teuthology.orchestra.run.smithi003.stdout: "bind": "/fake", 2024-06-01T02:24:08.046 INFO:teuthology.orchestra.run.smithi003.stdout: "fs": "foofs", 2024-06-01T02:24:08.047 INFO:teuthology.orchestra.run.smithi003.stdout: "path": "/", 2024-06-01T02:24:08.047 INFO:teuthology.orchestra.run.smithi003.stdout: "cluster": "foo", 2024-06-01T02:24:08.047 INFO:teuthology.orchestra.run.smithi003.stdout: "mode": "RW" 2024-06-01T02:24:08.047 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:24:08.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:08 smithi003 ceph-mon[25122]: pgmap v227: 97 pgs: 9 creating+peering, 9 unknown, 79 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:24:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:08 smithi003 ceph-mon[25122]: Creating key for client.nfs.foo.1.0.smithi027.dsjwox 2024-06-01T02:24:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-01T02:24:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-06-01T02:24:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:08 smithi003 ceph-mon[25122]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-06-01T02:24:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-01T02:24:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-01T02:24:08.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:08 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:08.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:08 smithi027 ceph-mon[27271]: pgmap v227: 97 pgs: 9 creating+peering, 9 unknown, 79 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-06-01T02:24:08.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:08.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:08 smithi027 ceph-mon[27271]: Creating key for client.nfs.foo.1.0.smithi027.dsjwox 2024-06-01T02:24:08.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-01T02:24:08.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-06-01T02:24:08.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:08 smithi027 ceph-mon[27271]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-06-01T02:24:08.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-01T02:24:08.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-01T02:24:08.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:08 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:08.750 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-06-01T02:24:09.287 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:09 smithi003 ceph-mon[25122]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:09.287 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:09 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-06-01T02:24:09.287 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:09 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-06-01T02:24:09.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:09 smithi027 ceph-mon[27271]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:09.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:09 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-06-01T02:24:09.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:09 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-06-01T02:24:10.283 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:10 smithi003 ceph-mon[25122]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-06-01T02:24:10.283 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-01T02:24:10.283 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-01T02:24:10.283 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-01T02:24:10.284 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-01T02:24:10.284 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:10.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:10 smithi027 ceph-mon[27271]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-06-01T02:24:10.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-01T02:24:10.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-01T02:24:10.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-01T02:24:10.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-01T02:24:10.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:11.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:11 smithi003 ceph-mon[25122]: Rados config object exists: conf-nfs.foo 2024-06-01T02:24:11.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:11 smithi003 ceph-mon[25122]: Creating key for client.nfs.foo.1.0.smithi027.dsjwox-rgw 2024-06-01T02:24:11.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:11 smithi003 ceph-mon[25122]: Deploying daemon nfs.foo.1.0.smithi027.dsjwox on smithi027 2024-06-01T02:24:11.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:11 smithi027 ceph-mon[27271]: Rados config object exists: conf-nfs.foo 2024-06-01T02:24:11.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:11 smithi027 ceph-mon[27271]: Creating key for client.nfs.foo.1.0.smithi027.dsjwox-rgw 2024-06-01T02:24:11.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:11 smithi027 ceph-mon[27271]: Deploying daemon nfs.foo.1.0.smithi027.dsjwox on smithi027 2024-06-01T02:24:12.097 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:12 smithi027 ceph-mon[27271]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:12.098 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:12 smithi027 ceph-mon[27271]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-06-01T02:24:12.312 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:12 smithi003 ceph-mon[25122]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:12.312 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:12 smithi003 ceph-mon[25122]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-06-01T02:24:13.534 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:13 smithi003 ceph-mon[25122]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:13.534 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:13 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:13.535 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:13 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:24:13.535 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:13 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:13.535 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:13 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:24:13.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:13 smithi027 ceph-mon[27271]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:13.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:13 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:13.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:13 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:24:13.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:13 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:13.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:13 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:24:14.567 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:14 smithi027 ceph-mon[27271]: pgmap v230: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-06-01T02:24:14.567 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:14 smithi027 ceph-mon[27271]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:14.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:14 smithi003 ceph-mon[25122]: pgmap v230: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-06-01T02:24:14.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:14 smithi003 ceph-mon[25122]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:16.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:15 smithi003 ceph-mon[25122]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:16.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:16.331 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:15 smithi003 ceph-mon[25122]: pgmap v231: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-06-01T02:24:16.332 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:15 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:16.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:15 smithi027 ceph-mon[27271]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:16.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:16.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:15 smithi027 ceph-mon[27271]: pgmap v231: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-06-01T02:24:16.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:15 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:17.308 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:16 smithi003 ceph-mon[25122]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:17.308 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo ?:2049 2/2 2s ago 13s count:2 2024-06-01T02:24:17.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:16 smithi027 ceph-mon[27271]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:18.044 INFO:teuthology.run_tasks:Running task vip.exec... 2024-06-01T02:24:18.054 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:24:18.055 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-06-01T02:24:18.084 INFO:teuthology.orchestra.run.smithi003.stderr:+ mkdir /mnt/foo 2024-06-01T02:24:18.100 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-06-01T02:24:18.168 INFO:teuthology.orchestra.run.smithi003.stderr:++ hostname 2024-06-01T02:24:18.168 INFO:teuthology.orchestra.run.smithi003.stderr:+ mount -t nfs smithi003:/fake /mnt/foo -o sync 2024-06-01T02:24:18.318 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-06-01T02:24:18.318 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.318 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.318 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.318 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:24:18.318 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:18.318 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:24:18.318 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.319 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.319 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:18 smithi003 ceph-mon[25122]: from='client.14600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-06-01T02:24:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:24:18.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:24:18.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:24:18.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:24:18.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:18 smithi027 ceph-mon[27271]: from='client.14600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:24:18.633 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-06-01T02:24:18.698 INFO:teuthology.orchestra.run.smithi003.stderr:+ echo test 2024-06-01T02:24:18.716 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-06-01T02:24:18.781 INFO:teuthology.orchestra.run.smithi003.stderr:+ sync 2024-06-01T02:24:19.034 INFO:teuthology.run_tasks:Running task parallel... 2024-06-01T02:24:19.044 INFO:teuthology.task.parallel:starting parallel... 2024-06-01T02:24:19.044 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-06-01T02:24:19.045 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-06-01T02:24:19.045 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:24:19.045 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-06-01T02:24:19.046 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-06-01T02:24:19.046 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-06-01T02:24:19.047 INFO:teuthology.task.exec:Executing custom commands... 2024-06-01T02:24:19.047 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:24:19.047 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-06-01T02:24:19.162 INFO:teuthology.orchestra.run.smithi003.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-06-01T02:24:19.162 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:24:19.162 INFO:teuthology.orchestra.run.smithi003.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-06-01T02:24:19.163 INFO:teuthology.orchestra.run.smithi003.stdout:3 of 5 processes prepared for launch 0 sec 2024-06-01T02:24:19.263 INFO:teuthology.orchestra.run.smithi003.stdout:5 of 5 processes prepared for launch 0 sec 2024-06-01T02:24:19.263 INFO:teuthology.orchestra.run.smithi003.stdout:releasing clients 2024-06-01T02:24:20.058 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:20 smithi003 ceph-mon[25122]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-06-01T02:24:20.263 INFO:teuthology.orchestra.run.smithi003.stdout: 5 299 55.15 MB/sec warmup 1 sec latency 11.149 ms 2024-06-01T02:24:20.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:20 smithi027 ceph-mon[27271]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-06-01T02:24:21.263 INFO:teuthology.orchestra.run.smithi003.stdout: 5 595 51.51 MB/sec warmup 2 sec latency 12.141 ms 2024-06-01T02:24:22.094 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:22 smithi003 ceph-mon[25122]: pgmap v234: 97 pgs: 97 active+clean; 16 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 MiB/s wr, 28 op/s 2024-06-01T02:24:22.263 INFO:teuthology.orchestra.run.smithi003.stdout: 5 1467 43.40 MB/sec warmup 3 sec latency 10.053 ms 2024-06-01T02:24:22.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:22 smithi027 ceph-mon[27271]: pgmap v234: 97 pgs: 97 active+clean; 16 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 MiB/s wr, 28 op/s 2024-06-01T02:24:23.263 INFO:teuthology.orchestra.run.smithi003.stdout: 5 2549 36.08 MB/sec warmup 4 sec latency 9.825 ms 2024-06-01T02:24:24.105 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:24 smithi003 ceph-mon[25122]: pgmap v235: 97 pgs: 97 active+clean; 84 MiB data, 127 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 7.4 MiB/s wr, 385 op/s 2024-06-01T02:24:24.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 3747 39.24 MB/sec warmup 5 sec latency 9.698 ms 2024-06-01T02:24:24.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:24 smithi027 ceph-mon[27271]: pgmap v235: 97 pgs: 97 active+clean; 84 MiB data, 127 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 7.4 MiB/s wr, 385 op/s 2024-06-01T02:24:25.109 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-06-01T02:24:25.109 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-06-01T02:24:25.109 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-06-01T02:24:25.109 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-06-01T02:24:25.109 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:24:25.109 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:24:25.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 4455 33.97 MB/sec warmup 6 sec latency 17.349 ms 2024-06-01T02:24:25.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-06-01T02:24:25.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-06-01T02:24:25.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-06-01T02:24:25.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-06-01T02:24:25.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:24:25.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:24:26.113 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:26 smithi003 ceph-mon[25122]: pgmap v236: 97 pgs: 97 active+clean; 84 MiB data, 131 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 7.4 MiB/s wr, 384 op/s 2024-06-01T02:24:26.113 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-06-01T02:24:26.113 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-06-01T02:24:26.113 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-06-01T02:24:26.113 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:26 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-06-01T02:24:26.113 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:26 smithi003 ceph-mon[25122]: osdmap e69: 8 total, 8 up, 8 in 2024-06-01T02:24:26.263 INFO:teuthology.orchestra.run.smithi003.stdout: 5 5407 32.49 MB/sec warmup 7 sec latency 39.284 ms 2024-06-01T02:24:26.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:26 smithi027 ceph-mon[27271]: pgmap v236: 97 pgs: 97 active+clean; 84 MiB data, 131 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 7.4 MiB/s wr, 384 op/s 2024-06-01T02:24:26.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-06-01T02:24:26.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-06-01T02:24:26.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-06-01T02:24:26.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:26 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-06-01T02:24:26.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:26 smithi027 ceph-mon[27271]: osdmap e69: 8 total, 8 up, 8 in 2024-06-01T02:24:27.119 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:27 smithi003 ceph-mon[25122]: osdmap e70: 8 total, 8 up, 8 in 2024-06-01T02:24:27.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 6312 31.30 MB/sec warmup 8 sec latency 12.538 ms 2024-06-01T02:24:27.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:27 smithi027 ceph-mon[27271]: osdmap e70: 8 total, 8 up, 8 in 2024-06-01T02:24:28.133 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:28 smithi003 ceph-mon[25122]: pgmap v239: 97 pgs: 97 active+clean; 129 MiB data, 207 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 19 MiB/s wr, 1.14k op/s 2024-06-01T02:24:28.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 7300 32.49 MB/sec warmup 9 sec latency 19.669 ms 2024-06-01T02:24:28.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:28 smithi027 ceph-mon[27271]: pgmap v239: 97 pgs: 97 active+clean; 129 MiB data, 207 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 19 MiB/s wr, 1.14k op/s 2024-06-01T02:24:29.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 8053 30.02 MB/sec warmup 10 sec latency 13.167 ms 2024-06-01T02:24:30.143 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:30 smithi003 ceph-mon[25122]: pgmap v240: 97 pgs: 97 active+clean; 139 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 20 MiB/s wr, 1.31k op/s 2024-06-01T02:24:30.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 9008 29.39 MB/sec warmup 11 sec latency 11.238 ms 2024-06-01T02:24:30.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:30 smithi027 ceph-mon[27271]: pgmap v240: 97 pgs: 97 active+clean; 139 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 20 MiB/s wr, 1.31k op/s 2024-06-01T02:24:31.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 9950 29.48 MB/sec warmup 12 sec latency 26.396 ms 2024-06-01T02:24:32.146 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:32 smithi003 ceph-mon[25122]: pgmap v241: 97 pgs: 97 active+clean; 144 MiB data, 296 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 14 MiB/s wr, 1.09k op/s 2024-06-01T02:24:32.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 10897 30.00 MB/sec warmup 13 sec latency 13.632 ms 2024-06-01T02:24:32.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:32 smithi027 ceph-mon[27271]: pgmap v241: 97 pgs: 97 active+clean; 144 MiB data, 296 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 14 MiB/s wr, 1.09k op/s 2024-06-01T02:24:33.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 11554 28.23 MB/sec warmup 14 sec latency 14.676 ms 2024-06-01T02:24:34.156 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:34 smithi003 ceph-mon[25122]: pgmap v242: 97 pgs: 97 active+clean; 143 MiB data, 366 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 18 MiB/s wr, 1.36k op/s 2024-06-01T02:24:34.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 12534 27.91 MB/sec warmup 15 sec latency 19.649 ms 2024-06-01T02:24:34.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:34 smithi027 ceph-mon[27271]: pgmap v242: 97 pgs: 97 active+clean; 143 MiB data, 366 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 18 MiB/s wr, 1.36k op/s 2024-06-01T02:24:35.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 13503 28.11 MB/sec warmup 16 sec latency 15.948 ms 2024-06-01T02:24:36.164 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:36 smithi003 ceph-mon[25122]: pgmap v243: 97 pgs: 97 active+clean; 143 MiB data, 367 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 1.11k op/s 2024-06-01T02:24:36.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 14418 28.47 MB/sec warmup 17 sec latency 19.251 ms 2024-06-01T02:24:36.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:36 smithi027 ceph-mon[27271]: pgmap v243: 97 pgs: 97 active+clean; 143 MiB data, 367 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 1.11k op/s 2024-06-01T02:24:37.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 15003 27.20 MB/sec warmup 18 sec latency 19.857 ms 2024-06-01T02:24:38.176 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:38 smithi003 ceph-mon[25122]: pgmap v244: 97 pgs: 97 active+clean; 153 MiB data, 455 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.03k op/s 2024-06-01T02:24:38.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 15798 27.01 MB/sec warmup 19 sec latency 19.781 ms 2024-06-01T02:24:38.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:38 smithi027 ceph-mon[27271]: pgmap v244: 97 pgs: 97 active+clean; 153 MiB data, 455 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.03k op/s 2024-06-01T02:24:39.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 16867 26.82 MB/sec warmup 20 sec latency 13.030 ms 2024-06-01T02:24:40.185 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:40 smithi003 ceph-mon[25122]: pgmap v245: 97 pgs: 97 active+clean; 158 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.00k op/s 2024-06-01T02:24:40.264 INFO:teuthology.orchestra.run.smithi003.stdout: 5 17939 27.58 MB/sec warmup 21 sec latency 17.512 ms 2024-06-01T02:24:40.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:40 smithi027 ceph-mon[27271]: pgmap v245: 97 pgs: 97 active+clean; 158 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.00k op/s 2024-06-01T02:24:41.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 18592 26.62 MB/sec warmup 22 sec latency 22.804 ms 2024-06-01T02:24:42.195 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:42 smithi003 ceph-mon[25122]: pgmap v246: 97 pgs: 97 active+clean; 170 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 13 MiB/s wr, 1.13k op/s 2024-06-01T02:24:42.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 19403 26.48 MB/sec warmup 23 sec latency 19.763 ms 2024-06-01T02:24:42.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:42 smithi027 ceph-mon[27271]: pgmap v246: 97 pgs: 97 active+clean; 170 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 13 MiB/s wr, 1.13k op/s 2024-06-01T02:24:43.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 20299 25.99 MB/sec warmup 24 sec latency 16.873 ms 2024-06-01T02:24:44.061 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:44 smithi003 ceph-mon[25122]: pgmap v247: 97 pgs: 97 active+clean; 175 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.13k op/s 2024-06-01T02:24:44.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 21356 26.74 MB/sec warmup 25 sec latency 22.163 ms 2024-06-01T02:24:44.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:44 smithi027 ceph-mon[27271]: pgmap v247: 97 pgs: 97 active+clean; 175 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.13k op/s 2024-06-01T02:24:45.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 22080 26.17 MB/sec warmup 26 sec latency 17.383 ms 2024-06-01T02:24:46.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 22874 26.09 MB/sec warmup 27 sec latency 16.840 ms 2024-06-01T02:24:46.268 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:46 smithi003 ceph-mon[25122]: pgmap v248: 97 pgs: 97 active+clean; 175 MiB data, 598 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 10 MiB/s wr, 952 op/s 2024-06-01T02:24:46.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:46 smithi027 ceph-mon[27271]: pgmap v248: 97 pgs: 97 active+clean; 175 MiB data, 598 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 10 MiB/s wr, 952 op/s 2024-06-01T02:24:47.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 23737 25.52 MB/sec warmup 28 sec latency 18.427 ms 2024-06-01T02:24:48.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 24857 26.32 MB/sec warmup 29 sec latency 18.608 ms 2024-06-01T02:24:48.277 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:48 smithi003 ceph-mon[25122]: pgmap v249: 97 pgs: 97 active+clean; 184 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.17k op/s 2024-06-01T02:24:48.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:48 smithi027 ceph-mon[27271]: pgmap v249: 97 pgs: 97 active+clean; 184 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.17k op/s 2024-06-01T02:24:49.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 25505 25.83 MB/sec warmup 30 sec latency 19.193 ms 2024-06-01T02:24:50.172 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:50 smithi003 ceph-mon[25122]: pgmap v250: 97 pgs: 97 active+clean; 189 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.10k op/s 2024-06-01T02:24:50.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 26367 25.80 MB/sec warmup 31 sec latency 20.580 ms 2024-06-01T02:24:50.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:50 smithi027 ceph-mon[27271]: pgmap v250: 97 pgs: 97 active+clean; 189 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.10k op/s 2024-06-01T02:24:51.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 27343 25.35 MB/sec warmup 32 sec latency 13.258 ms 2024-06-01T02:24:52.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 28360 25.93 MB/sec warmup 33 sec latency 21.552 ms 2024-06-01T02:24:52.304 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:52 smithi003 ceph-mon[25122]: pgmap v251: 97 pgs: 97 active+clean; 204 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 13 MiB/s wr, 1.27k op/s 2024-06-01T02:24:52.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:52 smithi027 ceph-mon[27271]: pgmap v251: 97 pgs: 97 active+clean; 204 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 13 MiB/s wr, 1.27k op/s 2024-06-01T02:24:53.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 29029 25.60 MB/sec warmup 34 sec latency 18.572 ms 2024-06-01T02:24:54.063 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:54 smithi003 ceph-mon[25122]: pgmap v252: 97 pgs: 97 active+clean; 206 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.22k op/s 2024-06-01T02:24:54.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 29936 25.59 MB/sec warmup 35 sec latency 15.463 ms 2024-06-01T02:24:54.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:54 smithi027 ceph-mon[27271]: pgmap v252: 97 pgs: 97 active+clean; 206 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.22k op/s 2024-06-01T02:24:55.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 30922 25.28 MB/sec warmup 36 sec latency 17.061 ms 2024-06-01T02:24:56.071 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:56 smithi003 ceph-mon[25122]: pgmap v253: 97 pgs: 97 active+clean; 206 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-06-01T02:24:56.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 31931 25.73 MB/sec warmup 37 sec latency 20.959 ms 2024-06-01T02:24:56.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:56 smithi027 ceph-mon[27271]: pgmap v253: 97 pgs: 97 active+clean; 206 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-06-01T02:24:57.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 32554 25.42 MB/sec warmup 38 sec latency 20.266 ms 2024-06-01T02:24:58.260 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:58 smithi003 ceph-mon[25122]: pgmap v254: 97 pgs: 97 active+clean; 215 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.34k op/s 2024-06-01T02:24:58.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 33323 25.19 MB/sec warmup 39 sec latency 18.841 ms 2024-06-01T02:24:58.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:58 smithi027 ceph-mon[27271]: pgmap v254: 97 pgs: 97 active+clean; 215 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.34k op/s 2024-06-01T02:24:59.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 34301 24.97 MB/sec warmup 40 sec latency 21.949 ms 2024-06-01T02:25:00.019 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:24:59 smithi003 ceph-mon[25122]: pgmap v255: 97 pgs: 97 active+clean; 218 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-06-01T02:25:00.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 35367 25.43 MB/sec warmup 41 sec latency 24.607 ms 2024-06-01T02:25:00.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:24:59 smithi027 ceph-mon[27271]: pgmap v255: 97 pgs: 97 active+clean; 218 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-06-01T02:25:01.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 36128 25.29 MB/sec warmup 42 sec latency 24.549 ms 2024-06-01T02:25:02.031 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:01 smithi003 ceph-mon[25122]: pgmap v256: 97 pgs: 97 active+clean; 217 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 13 MiB/s wr, 1.08k op/s 2024-06-01T02:25:02.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 36963 25.14 MB/sec warmup 43 sec latency 16.781 ms 2024-06-01T02:25:02.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:01 smithi027 ceph-mon[27271]: pgmap v256: 97 pgs: 97 active+clean; 217 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 13 MiB/s wr, 1.08k op/s 2024-06-01T02:25:03.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 37884 24.90 MB/sec warmup 44 sec latency 14.936 ms 2024-06-01T02:25:04.038 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:03 smithi003 ceph-mon[25122]: pgmap v257: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.09k op/s 2024-06-01T02:25:04.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 38713 25.14 MB/sec warmup 45 sec latency 42.917 ms 2024-06-01T02:25:04.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:03 smithi027 ceph-mon[27271]: pgmap v257: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.09k op/s 2024-06-01T02:25:05.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 39403 25.12 MB/sec warmup 46 sec latency 33.814 ms 2024-06-01T02:25:06.048 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:05 smithi003 ceph-mon[25122]: pgmap v258: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 10 MiB/s wr, 882 op/s 2024-06-01T02:25:06.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 40025 24.72 MB/sec warmup 47 sec latency 49.546 ms 2024-06-01T02:25:06.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:05 smithi027 ceph-mon[27271]: pgmap v258: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 10 MiB/s wr, 882 op/s 2024-06-01T02:25:07.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 40438 24.45 MB/sec warmup 48 sec latency 37.280 ms 2024-06-01T02:25:08.075 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:07 smithi003 ceph-mon[25122]: pgmap v259: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-06-01T02:25:08.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 41023 24.18 MB/sec warmup 49 sec latency 61.424 ms 2024-06-01T02:25:08.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:07 smithi027 ceph-mon[27271]: pgmap v259: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.23k op/s 2024-06-01T02:25:09.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 41474 23.85 MB/sec warmup 50 sec latency 37.910 ms 2024-06-01T02:25:10.106 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:09 smithi003 ceph-mon[25122]: pgmap v260: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 966 op/s 2024-06-01T02:25:10.265 INFO:teuthology.orchestra.run.smithi003.stdout: 5 41952 23.82 MB/sec warmup 51 sec latency 119.475 ms 2024-06-01T02:25:10.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:09 smithi027 ceph-mon[27271]: pgmap v260: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 966 op/s 2024-06-01T02:25:11.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 42625 23.90 MB/sec warmup 52 sec latency 113.713 ms 2024-06-01T02:25:12.122 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:11 smithi003 ceph-mon[25122]: pgmap v261: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 996 op/s 2024-06-01T02:25:12.168 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:11 smithi027 ceph-mon[27271]: pgmap v261: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 11 MiB/s wr, 996 op/s 2024-06-01T02:25:12.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 42972 23.61 MB/sec warmup 53 sec latency 64.755 ms 2024-06-01T02:25:13.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 43336 23.22 MB/sec warmup 54 sec latency 54.397 ms 2024-06-01T02:25:14.166 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:13 smithi003 ceph-mon[25122]: pgmap v262: 97 pgs: 97 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-06-01T02:25:14.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 43604 22.88 MB/sec warmup 55 sec latency 69.572 ms 2024-06-01T02:25:14.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:13 smithi027 ceph-mon[27271]: pgmap v262: 97 pgs: 97 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-06-01T02:25:15.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 43890 22.65 MB/sec warmup 56 sec latency 85.919 ms 2024-06-01T02:25:16.046 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:15 smithi003 ceph-mon[25122]: pgmap v263: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.1 MiB/s wr, 789 op/s 2024-06-01T02:25:16.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 44325 22.43 MB/sec warmup 57 sec latency 64.708 ms 2024-06-01T02:25:16.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:15 smithi027 ceph-mon[27271]: pgmap v263: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.1 MiB/s wr, 789 op/s 2024-06-01T02:25:17.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 44739 22.12 MB/sec warmup 58 sec latency 82.315 ms 2024-06-01T02:25:18.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:17 smithi003 ceph-mon[25122]: pgmap v264: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.7 MiB/s wr, 949 op/s 2024-06-01T02:25:18.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:17 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:25:18.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:17 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:25:18.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:17 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:25:18.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 45047 21.85 MB/sec warmup 59 sec latency 105.839 ms 2024-06-01T02:25:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:17 smithi027 ceph-mon[27271]: pgmap v264: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.7 MiB/s wr, 949 op/s 2024-06-01T02:25:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:17 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:25:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:17 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:25:18.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:17 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:25:19.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 45224 21.62 MB/sec warmup 60 sec latency 125.913 ms 2024-06-01T02:25:19.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:19 smithi003 ceph-mon[25122]: pgmap v265: 97 pgs: 97 active+clean; 245 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.0 MiB/s wr, 641 op/s 2024-06-01T02:25:20.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 45486 21.45 MB/sec warmup 61 sec latency 91.935 ms 2024-06-01T02:25:20.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:19 smithi027 ceph-mon[27271]: pgmap v265: 97 pgs: 97 active+clean; 245 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.0 MiB/s wr, 641 op/s 2024-06-01T02:25:21.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 45821 21.33 MB/sec warmup 62 sec latency 43.411 ms 2024-06-01T02:25:22.115 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:21 smithi003 ceph-mon[25122]: pgmap v266: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.1 MiB/s wr, 633 op/s 2024-06-01T02:25:22.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 46076 21.13 MB/sec warmup 63 sec latency 128.741 ms 2024-06-01T02:25:22.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:21 smithi027 ceph-mon[27271]: pgmap v266: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.1 MiB/s wr, 633 op/s 2024-06-01T02:25:23.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 46355 21.00 MB/sec warmup 64 sec latency 88.538 ms 2024-06-01T02:25:24.185 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:23 smithi003 ceph-mon[25122]: pgmap v267: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.0 MiB/s wr, 605 op/s 2024-06-01T02:25:24.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 46592 20.73 MB/sec warmup 65 sec latency 43.559 ms 2024-06-01T02:25:24.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:23 smithi027 ceph-mon[27271]: pgmap v267: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.0 MiB/s wr, 605 op/s 2024-06-01T02:25:24.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:25:24.949 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:25:25.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 46787 20.44 MB/sec warmup 66 sec latency 76.508 ms 2024-06-01T02:25:25.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:25:25.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:25:25.973 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:25 smithi003 ceph-mon[25122]: pgmap v268: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.2 MiB/s wr, 435 op/s 2024-06-01T02:25:26.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 47088 20.20 MB/sec warmup 67 sec latency 55.293 ms 2024-06-01T02:25:26.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:25 smithi027 ceph-mon[27271]: pgmap v268: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.2 MiB/s wr, 435 op/s 2024-06-01T02:25:27.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 47426 20.06 MB/sec warmup 68 sec latency 65.776 ms 2024-06-01T02:25:28.028 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:27 smithi003 ceph-mon[25122]: pgmap v269: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.1 MiB/s wr, 478 op/s 2024-06-01T02:25:28.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 47912 19.92 MB/sec warmup 69 sec latency 70.298 ms 2024-06-01T02:25:28.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:27 smithi027 ceph-mon[27271]: pgmap v269: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.1 MiB/s wr, 478 op/s 2024-06-01T02:25:29.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 48295 19.69 MB/sec warmup 70 sec latency 61.987 ms 2024-06-01T02:25:30.081 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:29 smithi003 ceph-mon[25122]: pgmap v270: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 381 op/s 2024-06-01T02:25:30.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 48615 19.51 MB/sec warmup 71 sec latency 84.050 ms 2024-06-01T02:25:30.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:29 smithi027 ceph-mon[27271]: pgmap v270: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 381 op/s 2024-06-01T02:25:31.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 48972 19.45 MB/sec warmup 72 sec latency 61.605 ms 2024-06-01T02:25:31.910 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:31 smithi003 ceph-mon[25122]: pgmap v271: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.3 MiB/s wr, 389 op/s 2024-06-01T02:25:32.266 INFO:teuthology.orchestra.run.smithi003.stdout: 5 49501 19.50 MB/sec warmup 73 sec latency 61.491 ms 2024-06-01T02:25:32.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:31 smithi027 ceph-mon[27271]: pgmap v271: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.3 MiB/s wr, 389 op/s 2024-06-01T02:25:33.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 49790 19.39 MB/sec warmup 74 sec latency 108.339 ms 2024-06-01T02:25:33.992 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:33 smithi003 ceph-mon[25122]: pgmap v272: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.2 MiB/s wr, 488 op/s 2024-06-01T02:25:34.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 49976 19.22 MB/sec warmup 75 sec latency 68.281 ms 2024-06-01T02:25:34.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:33 smithi027 ceph-mon[27271]: pgmap v272: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.2 MiB/s wr, 488 op/s 2024-06-01T02:25:35.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 50126 18.99 MB/sec warmup 76 sec latency 114.972 ms 2024-06-01T02:25:36.046 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:35 smithi003 ceph-mon[25122]: pgmap v273: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-06-01T02:25:36.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 50462 18.78 MB/sec warmup 77 sec latency 76.195 ms 2024-06-01T02:25:36.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:35 smithi027 ceph-mon[27271]: pgmap v273: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-06-01T02:25:37.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 50817 18.62 MB/sec warmup 78 sec latency 54.089 ms 2024-06-01T02:25:38.077 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:37 smithi003 ceph-mon[25122]: pgmap v274: 97 pgs: 97 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.6 MiB/s wr, 533 op/s 2024-06-01T02:25:38.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 51460 18.61 MB/sec warmup 79 sec latency 54.022 ms 2024-06-01T02:25:38.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:37 smithi027 ceph-mon[27271]: pgmap v274: 97 pgs: 97 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.6 MiB/s wr, 533 op/s 2024-06-01T02:25:39.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 51786 18.41 MB/sec warmup 80 sec latency 93.624 ms 2024-06-01T02:25:40.174 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:39 smithi003 ceph-mon[25122]: pgmap v275: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 461 op/s 2024-06-01T02:25:40.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 52025 18.24 MB/sec warmup 81 sec latency 256.053 ms 2024-06-01T02:25:40.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:39 smithi027 ceph-mon[27271]: pgmap v275: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 461 op/s 2024-06-01T02:25:41.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 52096 18.03 MB/sec warmup 82 sec latency 123.321 ms 2024-06-01T02:25:41.960 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:41 smithi003 ceph-mon[25122]: pgmap v276: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 473 op/s 2024-06-01T02:25:42.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 52142 17.82 MB/sec warmup 83 sec latency 137.031 ms 2024-06-01T02:25:42.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:41 smithi027 ceph-mon[27271]: pgmap v276: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 473 op/s 2024-06-01T02:25:43.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 52217 17.65 MB/sec warmup 84 sec latency 146.638 ms 2024-06-01T02:25:44.168 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:43 smithi003 ceph-mon[25122]: pgmap v277: 97 pgs: 97 active+clean; 272 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 523 op/s 2024-06-01T02:25:44.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 52386 17.56 MB/sec warmup 85 sec latency 113.550 ms 2024-06-01T02:25:44.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:43 smithi027 ceph-mon[27271]: pgmap v277: 97 pgs: 97 active+clean; 272 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 523 op/s 2024-06-01T02:25:45.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 52573 17.45 MB/sec warmup 86 sec latency 275.159 ms 2024-06-01T02:25:46.112 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:45 smithi003 ceph-mon[25122]: pgmap v278: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 409 op/s 2024-06-01T02:25:46.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 52847 17.36 MB/sec warmup 87 sec latency 308.660 ms 2024-06-01T02:25:46.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:45 smithi027 ceph-mon[27271]: pgmap v278: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 409 op/s 2024-06-01T02:25:47.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 53333 17.42 MB/sec warmup 88 sec latency 98.857 ms 2024-06-01T02:25:48.163 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:47 smithi003 ceph-mon[25122]: pgmap v279: 97 pgs: 97 active+clean; 279 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 424 op/s 2024-06-01T02:25:48.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 53620 17.28 MB/sec warmup 89 sec latency 83.539 ms 2024-06-01T02:25:48.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:47 smithi027 ceph-mon[27271]: pgmap v279: 97 pgs: 97 active+clean; 279 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 424 op/s 2024-06-01T02:25:49.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 53892 17.11 MB/sec warmup 90 sec latency 111.797 ms 2024-06-01T02:25:49.972 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:49 smithi003 ceph-mon[25122]: pgmap v280: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 314 op/s 2024-06-01T02:25:50.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 54179 16.97 MB/sec warmup 91 sec latency 104.108 ms 2024-06-01T02:25:50.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:49 smithi027 ceph-mon[27271]: pgmap v280: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 314 op/s 2024-06-01T02:25:51.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 54450 16.87 MB/sec warmup 92 sec latency 92.772 ms 2024-06-01T02:25:52.047 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:51 smithi003 ceph-mon[25122]: pgmap v281: 97 pgs: 97 active+clean; 291 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.8 MiB/s wr, 386 op/s 2024-06-01T02:25:52.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 54896 16.83 MB/sec warmup 93 sec latency 54.450 ms 2024-06-01T02:25:52.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:51 smithi027 ceph-mon[27271]: pgmap v281: 97 pgs: 97 active+clean; 291 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.8 MiB/s wr, 386 op/s 2024-06-01T02:25:53.267 INFO:teuthology.orchestra.run.smithi003.stdout: 5 55394 16.69 MB/sec warmup 94 sec latency 117.450 ms 2024-06-01T02:25:53.938 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:53 smithi003 ceph-mon[25122]: pgmap v282: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.1 MiB/s wr, 416 op/s 2024-06-01T02:25:54.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 55703 16.62 MB/sec warmup 95 sec latency 100.039 ms 2024-06-01T02:25:54.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:53 smithi027 ceph-mon[27271]: pgmap v282: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.1 MiB/s wr, 416 op/s 2024-06-01T02:25:55.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 56036 16.58 MB/sec warmup 96 sec latency 58.906 ms 2024-06-01T02:25:56.014 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:55 smithi003 ceph-mon[25122]: pgmap v283: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.4 MiB/s wr, 321 op/s 2024-06-01T02:25:56.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 56497 16.61 MB/sec warmup 97 sec latency 137.501 ms 2024-06-01T02:25:56.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:55 smithi027 ceph-mon[27271]: pgmap v283: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.4 MiB/s wr, 321 op/s 2024-06-01T02:25:57.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 57007 16.65 MB/sec warmup 98 sec latency 91.119 ms 2024-06-01T02:25:58.098 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:57 smithi003 ceph-mon[25122]: pgmap v284: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 351 op/s 2024-06-01T02:25:58.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 57286 16.51 MB/sec warmup 99 sec latency 161.697 ms 2024-06-01T02:25:58.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:57 smithi027 ceph-mon[27271]: pgmap v284: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 351 op/s 2024-06-01T02:25:59.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 57523 16.37 MB/sec warmup 100 sec latency 73.867 ms 2024-06-01T02:25:59.907 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:25:59 smithi003 ceph-mon[25122]: pgmap v285: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.1 MiB/s wr, 471 op/s 2024-06-01T02:26:00.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 57766 16.25 MB/sec warmup 101 sec latency 55.638 ms 2024-06-01T02:26:00.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:25:59 smithi027 ceph-mon[27271]: pgmap v285: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.1 MiB/s wr, 471 op/s 2024-06-01T02:26:01.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 58012 16.14 MB/sec warmup 102 sec latency 61.947 ms 2024-06-01T02:26:01.974 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:01 smithi003 ceph-mon[25122]: pgmap v286: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.4 MiB/s wr, 484 op/s 2024-06-01T02:26:02.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 58363 16.07 MB/sec warmup 103 sec latency 113.307 ms 2024-06-01T02:26:02.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:01 smithi027 ceph-mon[27271]: pgmap v286: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.4 MiB/s wr, 484 op/s 2024-06-01T02:26:03.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 58758 15.99 MB/sec warmup 104 sec latency 71.411 ms 2024-06-01T02:26:04.014 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:03 smithi003 ceph-mon[25122]: pgmap v287: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-06-01T02:26:04.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 59142 15.92 MB/sec warmup 105 sec latency 66.626 ms 2024-06-01T02:26:04.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:03 smithi027 ceph-mon[27271]: pgmap v287: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-06-01T02:26:05.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 59411 15.85 MB/sec warmup 106 sec latency 77.527 ms 2024-06-01T02:26:06.094 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:05 smithi003 ceph-mon[25122]: pgmap v288: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.4 MiB/s wr, 428 op/s 2024-06-01T02:26:06.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 59799 15.86 MB/sec warmup 107 sec latency 80.090 ms 2024-06-01T02:26:06.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:05 smithi027 ceph-mon[27271]: pgmap v288: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.4 MiB/s wr, 428 op/s 2024-06-01T02:26:07.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 60037 15.80 MB/sec warmup 108 sec latency 98.325 ms 2024-06-01T02:26:08.119 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:07 smithi003 ceph-mon[25122]: pgmap v289: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 578 op/s 2024-06-01T02:26:08.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 60457 15.80 MB/sec warmup 109 sec latency 80.462 ms 2024-06-01T02:26:08.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:07 smithi027 ceph-mon[27271]: pgmap v289: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 578 op/s 2024-06-01T02:26:09.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 60682 15.71 MB/sec warmup 110 sec latency 147.446 ms 2024-06-01T02:26:09.952 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:09 smithi003 ceph-mon[25122]: pgmap v290: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.2 MiB/s wr, 527 op/s 2024-06-01T02:26:10.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 60897 15.59 MB/sec warmup 111 sec latency 109.746 ms 2024-06-01T02:26:10.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:09 smithi027 ceph-mon[27271]: pgmap v290: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.2 MiB/s wr, 527 op/s 2024-06-01T02:26:11.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 61147 15.48 MB/sec warmup 112 sec latency 123.389 ms 2024-06-01T02:26:11.988 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:11 smithi003 ceph-mon[25122]: pgmap v291: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 424 op/s 2024-06-01T02:26:12.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 61422 15.40 MB/sec warmup 113 sec latency 51.568 ms 2024-06-01T02:26:12.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:11 smithi027 ceph-mon[27271]: pgmap v291: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 424 op/s 2024-06-01T02:26:13.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 61753 15.33 MB/sec warmup 114 sec latency 55.387 ms 2024-06-01T02:26:14.105 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:13 smithi003 ceph-mon[25122]: pgmap v292: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 561 op/s 2024-06-01T02:26:14.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 62167 15.29 MB/sec warmup 115 sec latency 62.624 ms 2024-06-01T02:26:14.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:13 smithi027 ceph-mon[27271]: pgmap v292: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 561 op/s 2024-06-01T02:26:15.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 62573 15.19 MB/sec warmup 116 sec latency 79.549 ms 2024-06-01T02:26:16.151 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:15 smithi003 ceph-mon[25122]: pgmap v293: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-06-01T02:26:16.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 62913 15.15 MB/sec warmup 117 sec latency 61.837 ms 2024-06-01T02:26:16.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:15 smithi027 ceph-mon[27271]: pgmap v293: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-06-01T02:26:17.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 63210 15.15 MB/sec warmup 118 sec latency 36.961 ms 2024-06-01T02:26:18.208 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:17 smithi003 ceph-mon[25122]: pgmap v294: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 426 op/s 2024-06-01T02:26:18.268 INFO:teuthology.orchestra.run.smithi003.stdout: 5 63629 15.15 MB/sec warmup 119 sec latency 45.793 ms 2024-06-01T02:26:18.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:17 smithi027 ceph-mon[27271]: pgmap v294: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 426 op/s 2024-06-01T02:26:19.956 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:19 smithi003 ceph-mon[25122]: pgmap v295: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 444 op/s 2024-06-01T02:26:20.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 64263 8.49 MB/sec execute 1 sec latency 88.424 ms 2024-06-01T02:26:20.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:19 smithi027 ceph-mon[27271]: pgmap v295: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 444 op/s 2024-06-01T02:26:21.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 64699 6.12 MB/sec execute 2 sec latency 86.571 ms 2024-06-01T02:26:22.013 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:21 smithi003 ceph-mon[25122]: pgmap v296: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 446 op/s 2024-06-01T02:26:22.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 64966 6.07 MB/sec execute 3 sec latency 100.497 ms 2024-06-01T02:26:22.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:21 smithi027 ceph-mon[27271]: pgmap v296: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 446 op/s 2024-06-01T02:26:23.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 65331 6.60 MB/sec execute 4 sec latency 69.034 ms 2024-06-01T02:26:24.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:23 smithi003 ceph-mon[25122]: pgmap v297: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.6 MiB/s wr, 601 op/s 2024-06-01T02:26:24.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 65776 7.51 MB/sec execute 5 sec latency 75.840 ms 2024-06-01T02:26:24.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:23 smithi027 ceph-mon[27271]: pgmap v297: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.6 MiB/s wr, 601 op/s 2024-06-01T02:26:25.171 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:26:25.172 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:26:25.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 66268 7.60 MB/sec execute 6 sec latency 109.950 ms 2024-06-01T02:26:25.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:26:25.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:26:25.982 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:25 smithi003 ceph-mon[25122]: pgmap v298: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 4.2 MiB/s wr, 440 op/s 2024-06-01T02:26:26.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 66559 7.98 MB/sec execute 7 sec latency 101.945 ms 2024-06-01T02:26:26.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:25 smithi027 ceph-mon[27271]: pgmap v298: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 4.2 MiB/s wr, 440 op/s 2024-06-01T02:26:27.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 66913 8.83 MB/sec execute 8 sec latency 109.852 ms 2024-06-01T02:26:28.218 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:27 smithi003 ceph-mon[25122]: pgmap v299: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 651 op/s 2024-06-01T02:26:28.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 67233 9.43 MB/sec execute 9 sec latency 126.646 ms 2024-06-01T02:26:28.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:27 smithi027 ceph-mon[27271]: pgmap v299: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 651 op/s 2024-06-01T02:26:29.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 67587 9.95 MB/sec execute 10 sec latency 88.666 ms 2024-06-01T02:26:30.003 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:29 smithi003 ceph-mon[25122]: pgmap v300: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.5 MiB/s wr, 649 op/s 2024-06-01T02:26:30.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 67922 9.53 MB/sec execute 11 sec latency 43.226 ms 2024-06-01T02:26:30.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:29 smithi027 ceph-mon[27271]: pgmap v300: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.5 MiB/s wr, 649 op/s 2024-06-01T02:26:31.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 68271 9.00 MB/sec execute 12 sec latency 74.098 ms 2024-06-01T02:26:32.036 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:31 smithi003 ceph-mon[25122]: pgmap v301: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 510 op/s 2024-06-01T02:26:32.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 68500 8.76 MB/sec execute 13 sec latency 76.003 ms 2024-06-01T02:26:32.347 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:31 smithi027 ceph-mon[27271]: pgmap v301: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 510 op/s 2024-06-01T02:26:33.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 68774 8.66 MB/sec execute 14 sec latency 105.833 ms 2024-06-01T02:26:34.082 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:33 smithi003 ceph-mon[25122]: pgmap v302: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.0 MiB/s wr, 657 op/s 2024-06-01T02:26:34.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 69072 8.61 MB/sec execute 15 sec latency 66.066 ms 2024-06-01T02:26:34.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:33 smithi027 ceph-mon[27271]: pgmap v302: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.0 MiB/s wr, 657 op/s 2024-06-01T02:26:35.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 69369 8.34 MB/sec execute 16 sec latency 125.762 ms 2024-06-01T02:26:36.163 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:35 smithi003 ceph-mon[25122]: pgmap v303: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.5 MiB/s wr, 471 op/s 2024-06-01T02:26:36.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 69562 7.96 MB/sec execute 17 sec latency 101.988 ms 2024-06-01T02:26:36.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:35 smithi027 ceph-mon[27271]: pgmap v303: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.5 MiB/s wr, 471 op/s 2024-06-01T02:26:37.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 69935 8.10 MB/sec execute 18 sec latency 123.422 ms 2024-06-01T02:26:38.214 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:37 smithi003 ceph-mon[25122]: pgmap v304: 97 pgs: 97 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 591 op/s 2024-06-01T02:26:38.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 70181 8.13 MB/sec execute 19 sec latency 112.263 ms 2024-06-01T02:26:38.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:37 smithi027 ceph-mon[27271]: pgmap v304: 97 pgs: 97 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 591 op/s 2024-06-01T02:26:39.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 70606 8.77 MB/sec execute 20 sec latency 41.895 ms 2024-06-01T02:26:40.054 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:39 smithi003 ceph-mon[25122]: pgmap v305: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 409 op/s 2024-06-01T02:26:40.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 71020 9.00 MB/sec execute 21 sec latency 92.995 ms 2024-06-01T02:26:40.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:39 smithi027 ceph-mon[27271]: pgmap v305: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 409 op/s 2024-06-01T02:26:41.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 71272 8.97 MB/sec execute 22 sec latency 154.311 ms 2024-06-01T02:26:42.131 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:41 smithi003 ceph-mon[25122]: pgmap v306: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-06-01T02:26:42.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 71532 8.74 MB/sec execute 23 sec latency 85.042 ms 2024-06-01T02:26:42.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:41 smithi027 ceph-mon[27271]: pgmap v306: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-06-01T02:26:43.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 71863 8.52 MB/sec execute 24 sec latency 105.417 ms 2024-06-01T02:26:43.977 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:43 smithi003 ceph-mon[25122]: pgmap v307: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 534 op/s 2024-06-01T02:26:44.269 INFO:teuthology.orchestra.run.smithi003.stdout: 5 72248 8.60 MB/sec execute 25 sec latency 42.512 ms 2024-06-01T02:26:44.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:43 smithi027 ceph-mon[27271]: pgmap v307: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 534 op/s 2024-06-01T02:26:45.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 72694 8.76 MB/sec execute 26 sec latency 84.526 ms 2024-06-01T02:26:46.029 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:45 smithi003 ceph-mon[25122]: pgmap v308: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 3.7 MiB/s wr, 358 op/s 2024-06-01T02:26:46.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 73253 8.61 MB/sec execute 27 sec latency 53.511 ms 2024-06-01T02:26:46.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:45 smithi027 ceph-mon[27271]: pgmap v308: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 3.7 MiB/s wr, 358 op/s 2024-06-01T02:26:47.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 73659 8.84 MB/sec execute 28 sec latency 50.266 ms 2024-06-01T02:26:48.076 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:47 smithi003 ceph-mon[25122]: pgmap v309: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.9 MiB/s wr, 516 op/s 2024-06-01T02:26:48.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 74153 9.28 MB/sec execute 29 sec latency 148.333 ms 2024-06-01T02:26:48.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:47 smithi027 ceph-mon[27271]: pgmap v309: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.9 MiB/s wr, 516 op/s 2024-06-01T02:26:49.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 74770 9.71 MB/sec execute 30 sec latency 208.278 ms 2024-06-01T02:26:50.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:49 smithi003 ceph-mon[25122]: pgmap v310: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 495 op/s 2024-06-01T02:26:50.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 75150 9.63 MB/sec execute 31 sec latency 97.518 ms 2024-06-01T02:26:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:49 smithi027 ceph-mon[27271]: pgmap v310: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 495 op/s 2024-06-01T02:26:51.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 75343 9.39 MB/sec execute 32 sec latency 69.434 ms 2024-06-01T02:26:52.146 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:51 smithi003 ceph-mon[25122]: pgmap v311: 97 pgs: 97 active+clean; 318 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 558 op/s 2024-06-01T02:26:52.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 75639 9.32 MB/sec execute 33 sec latency 81.025 ms 2024-06-01T02:26:52.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:51 smithi027 ceph-mon[27271]: pgmap v311: 97 pgs: 97 active+clean; 318 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 558 op/s 2024-06-01T02:26:53.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 75955 9.16 MB/sec execute 34 sec latency 92.981 ms 2024-06-01T02:26:53.932 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:53 smithi003 ceph-mon[25122]: pgmap v312: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.1 MiB/s wr, 544 op/s 2024-06-01T02:26:54.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 76307 9.21 MB/sec execute 35 sec latency 75.705 ms 2024-06-01T02:26:54.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:53 smithi027 ceph-mon[27271]: pgmap v312: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.1 MiB/s wr, 544 op/s 2024-06-01T02:26:55.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 76619 9.20 MB/sec execute 36 sec latency 88.221 ms 2024-06-01T02:26:55.984 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:55 smithi003 ceph-mon[25122]: pgmap v313: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 501 op/s 2024-06-01T02:26:56.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 77076 9.18 MB/sec execute 37 sec latency 94.311 ms 2024-06-01T02:26:56.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:55 smithi027 ceph-mon[27271]: pgmap v313: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 501 op/s 2024-06-01T02:26:57.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 77339 9.18 MB/sec execute 38 sec latency 77.458 ms 2024-06-01T02:26:58.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 77669 9.29 MB/sec execute 39 sec latency 50.071 ms 2024-06-01T02:26:58.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:58 smithi003 ceph-mon[25122]: pgmap v314: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.1 MiB/s wr, 635 op/s 2024-06-01T02:26:58.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:58 smithi003 ceph-mon[25122]: pgmap v315: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 576 op/s 2024-06-01T02:26:58.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:26:58.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:26:58.580 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:58 smithi003 ceph-mon[25122]: Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-01T02:26:58.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:58 smithi027 ceph-mon[27271]: pgmap v314: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.1 MiB/s wr, 635 op/s 2024-06-01T02:26:58.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:58 smithi027 ceph-mon[27271]: pgmap v315: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 576 op/s 2024-06-01T02:26:58.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:26:58.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:26:58.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:58 smithi027 ceph-mon[27271]: Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-01T02:26:59.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 77979 9.36 MB/sec execute 40 sec latency 90.050 ms 2024-06-01T02:27:00.036 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:26:59 smithi003 ceph-mon[25122]: pgmap v316: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.1 MiB/s wr, 466 op/s 2024-06-01T02:27:00.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 78256 9.36 MB/sec execute 41 sec latency 73.774 ms 2024-06-01T02:27:00.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:26:59 smithi027 ceph-mon[27271]: pgmap v316: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.1 MiB/s wr, 466 op/s 2024-06-01T02:27:01.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 78449 9.29 MB/sec execute 42 sec latency 106.820 ms 2024-06-01T02:27:02.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 78619 9.13 MB/sec execute 43 sec latency 138.996 ms 2024-06-01T02:27:02.634 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:02 smithi003 ceph-mon[25122]: pgmap v317: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 467 op/s 2024-06-01T02:27:02.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:02 smithi027 ceph-mon[27271]: pgmap v317: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 467 op/s 2024-06-01T02:27:03.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 78907 9.04 MB/sec execute 44 sec latency 125.417 ms 2024-06-01T02:27:04.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 79232 8.91 MB/sec execute 45 sec latency 118.439 ms 2024-06-01T02:27:04.674 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:04 smithi003 ceph-mon[25122]: pgmap v318: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 458 op/s 2024-06-01T02:27:04.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:04 smithi027 ceph-mon[27271]: pgmap v318: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 458 op/s 2024-06-01T02:27:05.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 79629 8.90 MB/sec execute 46 sec latency 76.560 ms 2024-06-01T02:27:06.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 80092 9.00 MB/sec execute 47 sec latency 50.878 ms 2024-06-01T02:27:06.489 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:06 smithi003 ceph-mon[25122]: pgmap v319: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.0 MiB/s wr, 477 op/s 2024-06-01T02:27:06.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:06 smithi027 ceph-mon[27271]: pgmap v319: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.0 MiB/s wr, 477 op/s 2024-06-01T02:27:07.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 80534 8.99 MB/sec execute 48 sec latency 58.710 ms 2024-06-01T02:27:08.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 80873 9.06 MB/sec execute 49 sec latency 89.116 ms 2024-06-01T02:27:08.518 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:08 smithi003 ceph-mon[25122]: pgmap v320: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 448 op/s 2024-06-01T02:27:08.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:08 smithi027 ceph-mon[27271]: pgmap v320: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 448 op/s 2024-06-01T02:27:09.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 81108 9.05 MB/sec execute 50 sec latency 75.714 ms 2024-06-01T02:27:10.093 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:09 smithi003 ceph-mon[25122]: pgmap v321: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.1 MiB/s wr, 374 op/s 2024-06-01T02:27:10.093 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:09 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:27:10.270 INFO:teuthology.orchestra.run.smithi003.stdout: 5 81378 9.11 MB/sec execute 51 sec latency 73.251 ms 2024-06-01T02:27:10.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:09 smithi027 ceph-mon[27271]: pgmap v321: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.1 MiB/s wr, 374 op/s 2024-06-01T02:27:10.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:09 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:27:11.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 81598 9.06 MB/sec execute 52 sec latency 97.862 ms 2024-06-01T02:27:12.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 81888 9.06 MB/sec execute 53 sec latency 70.318 ms 2024-06-01T02:27:12.657 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:12 smithi003 ceph-mon[25122]: pgmap v322: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 527 op/s 2024-06-01T02:27:12.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:12 smithi027 ceph-mon[27271]: pgmap v322: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 527 op/s 2024-06-01T02:27:13.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 82137 9.03 MB/sec execute 54 sec latency 101.567 ms 2024-06-01T02:27:14.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 82312 8.91 MB/sec execute 55 sec latency 68.954 ms 2024-06-01T02:27:14.459 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:14 smithi003 ceph-mon[25122]: pgmap v323: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 459 op/s 2024-06-01T02:27:14.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:14 smithi027 ceph-mon[27271]: pgmap v323: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 459 op/s 2024-06-01T02:27:15.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 82596 8.84 MB/sec execute 56 sec latency 84.203 ms 2024-06-01T02:27:16.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 83024 8.80 MB/sec execute 57 sec latency 102.110 ms 2024-06-01T02:27:16.490 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:16 smithi003 ceph-mon[25122]: pgmap v324: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.4 MiB/s wr, 421 op/s 2024-06-01T02:27:16.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:16 smithi027 ceph-mon[27271]: pgmap v324: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.4 MiB/s wr, 421 op/s 2024-06-01T02:27:17.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 83411 8.86 MB/sec execute 58 sec latency 82.015 ms 2024-06-01T02:27:18.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 83799 8.87 MB/sec execute 59 sec latency 40.467 ms 2024-06-01T02:27:18.540 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:18 smithi003 ceph-mon[25122]: pgmap v325: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.7 MiB/s wr, 499 op/s 2024-06-01T02:27:18.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:18 smithi027 ceph-mon[27271]: pgmap v325: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.7 MiB/s wr, 499 op/s 2024-06-01T02:27:19.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 84357 8.97 MB/sec execute 60 sec latency 64.720 ms 2024-06-01T02:27:20.052 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:19 smithi003 ceph-mon[25122]: pgmap v326: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 393 op/s 2024-06-01T02:27:20.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 84890 9.16 MB/sec execute 61 sec latency 75.152 ms 2024-06-01T02:27:20.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:19 smithi027 ceph-mon[27271]: pgmap v326: 97 pgs: 97 active+clean; 318 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 393 op/s 2024-06-01T02:27:21.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 85352 9.21 MB/sec execute 62 sec latency 151.657 ms 2024-06-01T02:27:22.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 85591 9.19 MB/sec execute 63 sec latency 53.955 ms 2024-06-01T02:27:22.618 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:22 smithi003 ceph-mon[25122]: pgmap v327: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 547 op/s 2024-06-01T02:27:22.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:22 smithi027 ceph-mon[27271]: pgmap v327: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 547 op/s 2024-06-01T02:27:23.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 85837 9.18 MB/sec execute 64 sec latency 127.357 ms 2024-06-01T02:27:24.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 86142 9.09 MB/sec execute 65 sec latency 55.346 ms 2024-06-01T02:27:24.634 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:24 smithi003 ceph-mon[25122]: pgmap v328: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.1 MiB/s wr, 441 op/s 2024-06-01T02:27:24.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:24 smithi027 ceph-mon[27271]: pgmap v328: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.1 MiB/s wr, 441 op/s 2024-06-01T02:27:25.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 86482 9.03 MB/sec execute 66 sec latency 93.995 ms 2024-06-01T02:27:25.646 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:27:25.646 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:27:25.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:27:25.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:27:26.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 86746 8.97 MB/sec execute 67 sec latency 65.376 ms 2024-06-01T02:27:26.660 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:26 smithi003 ceph-mon[25122]: pgmap v329: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.4 MiB/s wr, 506 op/s 2024-06-01T02:27:26.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:26 smithi027 ceph-mon[27271]: pgmap v329: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.4 MiB/s wr, 506 op/s 2024-06-01T02:27:27.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 87015 9.02 MB/sec execute 68 sec latency 75.077 ms 2024-06-01T02:27:28.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 87505 9.05 MB/sec execute 69 sec latency 51.820 ms 2024-06-01T02:27:28.697 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:28 smithi003 ceph-mon[25122]: pgmap v330: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 600 op/s 2024-06-01T02:27:28.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:28 smithi027 ceph-mon[27271]: pgmap v330: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 600 op/s 2024-06-01T02:27:29.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 88051 9.10 MB/sec execute 70 sec latency 62.161 ms 2024-06-01T02:27:29.958 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:29 smithi003 ceph-mon[25122]: pgmap v331: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-06-01T02:27:30.271 INFO:teuthology.orchestra.run.smithi003.stdout: 5 88524 9.26 MB/sec execute 71 sec latency 119.506 ms 2024-06-01T02:27:30.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:29 smithi027 ceph-mon[27271]: pgmap v331: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.9 MiB/s wr, 486 op/s 2024-06-01T02:27:31.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 88946 9.28 MB/sec execute 72 sec latency 54.386 ms 2024-06-01T02:27:32.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 89241 9.30 MB/sec execute 73 sec latency 129.273 ms 2024-06-01T02:27:32.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:32 smithi003 ceph-mon[25122]: pgmap v332: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 598 op/s 2024-06-01T02:27:32.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:32 smithi027 ceph-mon[27271]: pgmap v332: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 598 op/s 2024-06-01T02:27:33.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 89538 9.27 MB/sec execute 74 sec latency 97.090 ms 2024-06-01T02:27:34.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 89698 9.15 MB/sec execute 75 sec latency 129.728 ms 2024-06-01T02:27:34.601 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:34 smithi003 ceph-mon[25122]: pgmap v333: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.6 MiB/s wr, 499 op/s 2024-06-01T02:27:34.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:34 smithi027 ceph-mon[27271]: pgmap v333: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.6 MiB/s wr, 499 op/s 2024-06-01T02:27:35.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 90020 9.10 MB/sec execute 76 sec latency 73.310 ms 2024-06-01T02:27:36.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 90364 9.10 MB/sec execute 77 sec latency 70.719 ms 2024-06-01T02:27:36.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:36 smithi003 ceph-mon[25122]: pgmap v334: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.5 MiB/s wr, 544 op/s 2024-06-01T02:27:36.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:36 smithi027 ceph-mon[27271]: pgmap v334: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.5 MiB/s wr, 544 op/s 2024-06-01T02:27:37.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 90720 9.10 MB/sec execute 78 sec latency 107.372 ms 2024-06-01T02:27:38.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 91206 9.13 MB/sec execute 79 sec latency 45.454 ms 2024-06-01T02:27:38.479 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:38 smithi003 ceph-mon[25122]: pgmap v335: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 547 op/s 2024-06-01T02:27:38.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:38 smithi027 ceph-mon[27271]: pgmap v335: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 547 op/s 2024-06-01T02:27:39.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 91555 9.12 MB/sec execute 80 sec latency 88.419 ms 2024-06-01T02:27:39.989 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:39 smithi003 ceph-mon[25122]: pgmap v336: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 395 op/s 2024-06-01T02:27:40.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 91924 9.24 MB/sec execute 81 sec latency 85.157 ms 2024-06-01T02:27:40.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:39 smithi027 ceph-mon[27271]: pgmap v336: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 395 op/s 2024-06-01T02:27:41.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 92243 9.25 MB/sec execute 82 sec latency 62.916 ms 2024-06-01T02:27:42.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 92574 9.25 MB/sec execute 83 sec latency 70.015 ms 2024-06-01T02:27:42.529 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:42 smithi003 ceph-mon[25122]: pgmap v337: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 556 op/s 2024-06-01T02:27:42.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:42 smithi027 ceph-mon[27271]: pgmap v337: 97 pgs: 97 active+clean; 324 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 556 op/s 2024-06-01T02:27:43.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 92815 9.23 MB/sec execute 84 sec latency 92.843 ms 2024-06-01T02:27:44.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 92978 9.15 MB/sec execute 85 sec latency 100.711 ms 2024-06-01T02:27:44.564 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:44 smithi003 ceph-mon[25122]: pgmap v338: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 497 op/s 2024-06-01T02:27:44.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:44 smithi027 ceph-mon[27271]: pgmap v338: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 497 op/s 2024-06-01T02:27:45.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 93259 9.13 MB/sec execute 86 sec latency 80.359 ms 2024-06-01T02:27:46.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 93609 9.06 MB/sec execute 87 sec latency 122.572 ms 2024-06-01T02:27:46.596 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:46 smithi003 ceph-mon[25122]: pgmap v339: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.9 MiB/s wr, 483 op/s 2024-06-01T02:27:46.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:46 smithi027 ceph-mon[27271]: pgmap v339: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.9 MiB/s wr, 483 op/s 2024-06-01T02:27:47.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 93918 9.05 MB/sec execute 88 sec latency 110.092 ms 2024-06-01T02:27:48.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 94156 9.00 MB/sec execute 89 sec latency 67.232 ms 2024-06-01T02:27:48.644 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:48 smithi003 ceph-mon[25122]: pgmap v340: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 527 op/s 2024-06-01T02:27:48.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:48 smithi027 ceph-mon[27271]: pgmap v340: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 527 op/s 2024-06-01T02:27:49.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 94528 9.00 MB/sec execute 90 sec latency 66.664 ms 2024-06-01T02:27:49.963 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:49 smithi003 ceph-mon[25122]: pgmap v341: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-06-01T02:27:50.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 94872 9.01 MB/sec execute 91 sec latency 70.394 ms 2024-06-01T02:27:50.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:49 smithi027 ceph-mon[27271]: pgmap v341: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-06-01T02:27:51.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 95100 8.97 MB/sec execute 92 sec latency 103.742 ms 2024-06-01T02:27:52.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 95442 9.07 MB/sec execute 93 sec latency 64.803 ms 2024-06-01T02:27:52.585 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:52 smithi003 ceph-mon[25122]: pgmap v342: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 550 op/s 2024-06-01T02:27:52.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:52 smithi027 ceph-mon[27271]: pgmap v342: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 550 op/s 2024-06-01T02:27:53.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 95699 9.06 MB/sec execute 94 sec latency 168.553 ms 2024-06-01T02:27:54.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 96003 9.07 MB/sec execute 95 sec latency 47.091 ms 2024-06-01T02:27:54.625 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:54 smithi003 ceph-mon[25122]: pgmap v343: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 424 op/s 2024-06-01T02:27:54.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:54 smithi027 ceph-mon[27271]: pgmap v343: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 424 op/s 2024-06-01T02:27:55.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 96243 9.05 MB/sec execute 96 sec latency 61.459 ms 2024-06-01T02:27:56.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 96409 8.99 MB/sec execute 97 sec latency 98.449 ms 2024-06-01T02:27:56.690 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:56 smithi003 ceph-mon[25122]: pgmap v344: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.2 MiB/s wr, 389 op/s 2024-06-01T02:27:56.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:56 smithi027 ceph-mon[27271]: pgmap v344: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.2 MiB/s wr, 389 op/s 2024-06-01T02:27:57.272 INFO:teuthology.orchestra.run.smithi003.stdout: 5 96636 8.92 MB/sec execute 98 sec latency 81.183 ms 2024-06-01T02:27:57.700 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:57 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:27:57.700 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:57 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:27:57.700 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:57 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:27:57.742 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:57 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:27:57.742 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:57 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:27:57.742 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:57 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:27:58.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 96976 8.91 MB/sec execute 99 sec latency 72.681 ms 2024-06-01T02:27:58.726 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:58 smithi003 ceph-mon[25122]: pgmap v345: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 466 op/s 2024-06-01T02:27:58.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:58 smithi027 ceph-mon[27271]: pgmap v345: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 466 op/s 2024-06-01T02:27:59.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 97346 8.87 MB/sec execute 100 sec latency 88.203 ms 2024-06-01T02:28:00.018 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:27:59 smithi003 ceph-mon[25122]: pgmap v346: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 328 op/s 2024-06-01T02:28:00.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 97593 8.87 MB/sec execute 101 sec latency 59.491 ms 2024-06-01T02:28:00.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:27:59 smithi027 ceph-mon[27271]: pgmap v346: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 328 op/s 2024-06-01T02:28:01.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 97975 8.87 MB/sec execute 102 sec latency 52.277 ms 2024-06-01T02:28:02.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 98321 8.87 MB/sec execute 103 sec latency 48.239 ms 2024-06-01T02:28:02.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:02 smithi003 ceph-mon[25122]: pgmap v347: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 433 op/s 2024-06-01T02:28:02.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:02 smithi027 ceph-mon[27271]: pgmap v347: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 433 op/s 2024-06-01T02:28:03.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 98626 8.86 MB/sec execute 104 sec latency 97.854 ms 2024-06-01T02:28:04.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 98960 8.93 MB/sec execute 105 sec latency 90.280 ms 2024-06-01T02:28:04.648 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:04 smithi003 ceph-mon[25122]: pgmap v348: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 415 op/s 2024-06-01T02:28:04.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:04 smithi027 ceph-mon[27271]: pgmap v348: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 415 op/s 2024-06-01T02:28:05.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 99236 8.93 MB/sec execute 106 sec latency 81.232 ms 2024-06-01T02:28:06.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 99495 8.94 MB/sec execute 107 sec latency 64.273 ms 2024-06-01T02:28:06.667 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:06 smithi003 ceph-mon[25122]: pgmap v349: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 437 op/s 2024-06-01T02:28:06.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:06 smithi027 ceph-mon[27271]: pgmap v349: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 437 op/s 2024-06-01T02:28:07.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 99761 8.92 MB/sec execute 108 sec latency 76.333 ms 2024-06-01T02:28:08.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 99949 8.87 MB/sec execute 109 sec latency 58.702 ms 2024-06-01T02:28:08.508 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:08 smithi003 ceph-mon[25122]: pgmap v350: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 468 op/s 2024-06-01T02:28:08.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:08 smithi027 ceph-mon[27271]: pgmap v350: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 468 op/s 2024-06-01T02:28:09.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 100162 8.82 MB/sec execute 110 sec latency 76.012 ms 2024-06-01T02:28:10.048 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:09 smithi003 ceph-mon[25122]: pgmap v351: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 347 op/s 2024-06-01T02:28:10.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 100476 8.80 MB/sec execute 111 sec latency 61.923 ms 2024-06-01T02:28:10.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:09 smithi027 ceph-mon[27271]: pgmap v351: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 347 op/s 2024-06-01T02:28:11.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 100754 8.75 MB/sec execute 112 sec latency 101.943 ms 2024-06-01T02:28:12.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 101009 8.72 MB/sec execute 113 sec latency 85.035 ms 2024-06-01T02:28:12.695 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:12 smithi003 ceph-mon[25122]: pgmap v352: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-06-01T02:28:12.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:12 smithi027 ceph-mon[27271]: pgmap v352: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-06-01T02:28:13.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 101234 8.71 MB/sec execute 114 sec latency 51.167 ms 2024-06-01T02:28:14.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 101546 8.70 MB/sec execute 115 sec latency 76.895 ms 2024-06-01T02:28:14.728 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:14 smithi003 ceph-mon[25122]: pgmap v353: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 411 op/s 2024-06-01T02:28:14.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:14 smithi027 ceph-mon[27271]: pgmap v353: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 411 op/s 2024-06-01T02:28:15.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 101859 8.70 MB/sec execute 116 sec latency 82.521 ms 2024-06-01T02:28:16.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 102132 8.68 MB/sec execute 117 sec latency 55.489 ms 2024-06-01T02:28:16.577 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:16 smithi003 ceph-mon[25122]: pgmap v354: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 426 op/s 2024-06-01T02:28:16.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:16 smithi027 ceph-mon[27271]: pgmap v354: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 426 op/s 2024-06-01T02:28:17.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 102428 8.72 MB/sec execute 118 sec latency 102.124 ms 2024-06-01T02:28:18.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 102762 8.76 MB/sec execute 119 sec latency 68.544 ms 2024-06-01T02:28:18.680 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:18 smithi003 ceph-mon[25122]: pgmap v355: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-06-01T02:28:18.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:18 smithi027 ceph-mon[27271]: pgmap v355: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-06-01T02:28:19.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 103027 8.75 MB/sec execute 120 sec latency 278.586 ms 2024-06-01T02:28:19.969 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:19 smithi003 ceph-mon[25122]: pgmap v356: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.3 MiB/s wr, 338 op/s 2024-06-01T02:28:20.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 103266 8.75 MB/sec execute 121 sec latency 182.809 ms 2024-06-01T02:28:20.348 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:19 smithi027 ceph-mon[27271]: pgmap v356: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.3 MiB/s wr, 338 op/s 2024-06-01T02:28:21.273 INFO:teuthology.orchestra.run.smithi003.stdout: 5 103491 8.71 MB/sec execute 122 sec latency 104.943 ms 2024-06-01T02:28:22.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 103755 8.69 MB/sec execute 123 sec latency 57.507 ms 2024-06-01T02:28:22.563 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:22 smithi003 ceph-mon[25122]: pgmap v357: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 495 op/s 2024-06-01T02:28:22.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:22 smithi027 ceph-mon[27271]: pgmap v357: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 495 op/s 2024-06-01T02:28:23.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 104102 8.66 MB/sec execute 124 sec latency 75.328 ms 2024-06-01T02:28:24.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 104498 8.65 MB/sec execute 125 sec latency 59.120 ms 2024-06-01T02:28:24.604 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:24 smithi003 ceph-mon[25122]: pgmap v358: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-06-01T02:28:24.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:24 smithi027 ceph-mon[27271]: pgmap v358: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-06-01T02:28:25.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 104783 8.64 MB/sec execute 126 sec latency 81.739 ms 2024-06-01T02:28:25.612 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:28:25.613 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:28:25.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:28:25.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:28:26.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 105167 8.64 MB/sec execute 127 sec latency 80.299 ms 2024-06-01T02:28:26.637 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:26 smithi003 ceph-mon[25122]: pgmap v359: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.1 MiB/s wr, 365 op/s 2024-06-01T02:28:26.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:26 smithi027 ceph-mon[27271]: pgmap v359: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.1 MiB/s wr, 365 op/s 2024-06-01T02:28:27.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 105527 8.65 MB/sec execute 128 sec latency 60.980 ms 2024-06-01T02:28:28.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 105868 8.67 MB/sec execute 129 sec latency 82.204 ms 2024-06-01T02:28:28.578 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:28 smithi003 ceph-mon[25122]: pgmap v360: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-06-01T02:28:28.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:28 smithi027 ceph-mon[27271]: pgmap v360: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-06-01T02:28:29.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 106116 8.70 MB/sec execute 130 sec latency 67.326 ms 2024-06-01T02:28:29.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:29 smithi003 ceph-mon[25122]: pgmap v361: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 394 op/s 2024-06-01T02:28:30.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 106394 8.71 MB/sec execute 131 sec latency 126.605 ms 2024-06-01T02:28:30.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:29 smithi027 ceph-mon[27271]: pgmap v361: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 394 op/s 2024-06-01T02:28:31.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 106596 8.69 MB/sec execute 132 sec latency 105.791 ms 2024-06-01T02:28:32.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 106731 8.65 MB/sec execute 133 sec latency 134.346 ms 2024-06-01T02:28:32.677 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:32 smithi003 ceph-mon[25122]: pgmap v362: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 533 op/s 2024-06-01T02:28:32.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:32 smithi027 ceph-mon[27271]: pgmap v362: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 533 op/s 2024-06-01T02:28:33.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 106855 8.62 MB/sec execute 134 sec latency 210.522 ms 2024-06-01T02:28:34.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 106927 8.56 MB/sec execute 135 sec latency 138.352 ms 2024-06-01T02:28:34.493 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:34 smithi003 ceph-mon[25122]: pgmap v363: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 390 op/s 2024-06-01T02:28:34.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:34 smithi027 ceph-mon[27271]: pgmap v363: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 390 op/s 2024-06-01T02:28:35.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 107058 8.52 MB/sec execute 136 sec latency 117.849 ms 2024-06-01T02:28:36.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 107304 8.50 MB/sec execute 137 sec latency 124.951 ms 2024-06-01T02:28:36.543 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:36 smithi003 ceph-mon[25122]: pgmap v364: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-06-01T02:28:36.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:36 smithi027 ceph-mon[27271]: pgmap v364: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-06-01T02:28:37.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 107614 8.47 MB/sec execute 138 sec latency 116.663 ms 2024-06-01T02:28:38.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 107898 8.44 MB/sec execute 139 sec latency 72.753 ms 2024-06-01T02:28:38.603 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:38 smithi003 ceph-mon[25122]: pgmap v365: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.1 MiB/s wr, 412 op/s 2024-06-01T02:28:38.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:38 smithi027 ceph-mon[27271]: pgmap v365: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.1 MiB/s wr, 412 op/s 2024-06-01T02:28:39.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 108181 8.45 MB/sec execute 140 sec latency 55.690 ms 2024-06-01T02:28:40.183 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:39 smithi003 ceph-mon[25122]: pgmap v366: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.0 MiB/s wr, 235 op/s 2024-06-01T02:28:40.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 108378 8.40 MB/sec execute 141 sec latency 83.233 ms 2024-06-01T02:28:40.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:39 smithi027 ceph-mon[27271]: pgmap v366: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.0 MiB/s wr, 235 op/s 2024-06-01T02:28:41.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 108836 8.43 MB/sec execute 142 sec latency 104.852 ms 2024-06-01T02:28:42.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 109136 8.42 MB/sec execute 143 sec latency 83.498 ms 2024-06-01T02:28:42.494 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:42 smithi003 ceph-mon[25122]: pgmap v367: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-06-01T02:28:42.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:42 smithi027 ceph-mon[27271]: pgmap v367: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-06-01T02:28:43.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 109350 8.43 MB/sec execute 144 sec latency 73.687 ms 2024-06-01T02:28:44.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 109596 8.45 MB/sec execute 145 sec latency 89.243 ms 2024-06-01T02:28:44.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:44 smithi003 ceph-mon[25122]: pgmap v368: 97 pgs: 97 active+clean; 323 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.9 MiB/s wr, 291 op/s 2024-06-01T02:28:44.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:44 smithi027 ceph-mon[27271]: pgmap v368: 97 pgs: 97 active+clean; 323 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.9 MiB/s wr, 291 op/s 2024-06-01T02:28:45.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 109866 8.45 MB/sec execute 146 sec latency 63.620 ms 2024-06-01T02:28:46.274 INFO:teuthology.orchestra.run.smithi003.stdout: 5 110166 8.46 MB/sec execute 147 sec latency 146.937 ms 2024-06-01T02:28:46.646 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:46 smithi003 ceph-mon[25122]: pgmap v369: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.3 MiB/s wr, 376 op/s 2024-06-01T02:28:46.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:46 smithi027 ceph-mon[27271]: pgmap v369: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.3 MiB/s wr, 376 op/s 2024-06-01T02:28:47.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 110411 8.45 MB/sec execute 148 sec latency 60.939 ms 2024-06-01T02:28:48.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 110615 8.42 MB/sec execute 149 sec latency 98.830 ms 2024-06-01T02:28:48.677 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:48 smithi003 ceph-mon[25122]: pgmap v370: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 362 op/s 2024-06-01T02:28:48.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:48 smithi027 ceph-mon[27271]: pgmap v370: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 362 op/s 2024-06-01T02:28:49.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 110844 8.40 MB/sec execute 150 sec latency 95.984 ms 2024-06-01T02:28:49.969 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:49 smithi003 ceph-mon[25122]: pgmap v371: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.3 MiB/s wr, 341 op/s 2024-06-01T02:28:50.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 111094 8.35 MB/sec execute 151 sec latency 102.209 ms 2024-06-01T02:28:50.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:49 smithi027 ceph-mon[27271]: pgmap v371: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.3 MiB/s wr, 341 op/s 2024-06-01T02:28:51.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 111343 8.33 MB/sec execute 152 sec latency 84.954 ms 2024-06-01T02:28:52.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 111616 8.32 MB/sec execute 153 sec latency 61.799 ms 2024-06-01T02:28:52.570 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:52 smithi003 ceph-mon[25122]: pgmap v372: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 475 op/s 2024-06-01T02:28:52.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:52 smithi027 ceph-mon[27271]: pgmap v372: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 475 op/s 2024-06-01T02:28:53.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 111927 8.31 MB/sec execute 154 sec latency 58.997 ms 2024-06-01T02:28:54.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 112304 8.32 MB/sec execute 155 sec latency 94.601 ms 2024-06-01T02:28:54.538 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:54 smithi003 ceph-mon[25122]: pgmap v373: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 338 op/s 2024-06-01T02:28:54.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:54 smithi027 ceph-mon[27271]: pgmap v373: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 338 op/s 2024-06-01T02:28:55.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 112683 8.33 MB/sec execute 156 sec latency 70.926 ms 2024-06-01T02:28:56.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 113016 8.37 MB/sec execute 157 sec latency 62.544 ms 2024-06-01T02:28:56.568 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:56 smithi003 ceph-mon[25122]: pgmap v374: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 396 op/s 2024-06-01T02:28:56.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:56 smithi027 ceph-mon[27271]: pgmap v374: 97 pgs: 97 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 396 op/s 2024-06-01T02:28:57.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 113255 8.38 MB/sec execute 158 sec latency 98.830 ms 2024-06-01T02:28:58.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 113537 8.39 MB/sec execute 159 sec latency 120.258 ms 2024-06-01T02:28:58.627 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:58 smithi003 ceph-mon[25122]: pgmap v375: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 407 op/s 2024-06-01T02:28:58.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:58 smithi027 ceph-mon[27271]: pgmap v375: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 407 op/s 2024-06-01T02:28:59.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 113739 8.37 MB/sec execute 160 sec latency 94.971 ms 2024-06-01T02:29:00.196 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:28:59 smithi003 ceph-mon[25122]: pgmap v376: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.5 MiB/s wr, 366 op/s 2024-06-01T02:29:00.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 113906 8.34 MB/sec execute 161 sec latency 75.451 ms 2024-06-01T02:29:00.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:28:59 smithi027 ceph-mon[27271]: pgmap v376: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.5 MiB/s wr, 366 op/s 2024-06-01T02:29:01.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 114002 8.30 MB/sec execute 162 sec latency 200.978 ms 2024-06-01T02:29:02.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 114162 8.28 MB/sec execute 163 sec latency 259.840 ms 2024-06-01T02:29:02.606 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:02 smithi003 ceph-mon[25122]: pgmap v377: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 484 op/s 2024-06-01T02:29:02.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:02 smithi027 ceph-mon[27271]: pgmap v377: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 484 op/s 2024-06-01T02:29:03.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 114469 8.27 MB/sec execute 164 sec latency 67.330 ms 2024-06-01T02:29:04.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 114752 8.24 MB/sec execute 165 sec latency 55.097 ms 2024-06-01T02:29:04.673 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:04 smithi003 ceph-mon[25122]: pgmap v378: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 367 op/s 2024-06-01T02:29:04.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:04 smithi027 ceph-mon[27271]: pgmap v378: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 367 op/s 2024-06-01T02:29:05.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 115039 8.22 MB/sec execute 166 sec latency 67.721 ms 2024-06-01T02:29:06.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 115315 8.23 MB/sec execute 167 sec latency 150.647 ms 2024-06-01T02:29:06.781 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:06 smithi003 ceph-mon[25122]: pgmap v379: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.0 MiB/s wr, 359 op/s 2024-06-01T02:29:06.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:06 smithi027 ceph-mon[27271]: pgmap v379: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.0 MiB/s wr, 359 op/s 2024-06-01T02:29:07.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 115628 8.22 MB/sec execute 168 sec latency 91.266 ms 2024-06-01T02:29:08.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 116044 8.25 MB/sec execute 169 sec latency 86.434 ms 2024-06-01T02:29:08.621 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:08 smithi003 ceph-mon[25122]: pgmap v380: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 409 op/s 2024-06-01T02:29:08.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:08 smithi027 ceph-mon[27271]: pgmap v380: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 409 op/s 2024-06-01T02:29:09.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 116363 8.24 MB/sec execute 170 sec latency 85.046 ms 2024-06-01T02:29:10.167 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:09 smithi003 ceph-mon[25122]: pgmap v381: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.0 MiB/s wr, 293 op/s 2024-06-01T02:29:10.275 INFO:teuthology.orchestra.run.smithi003.stdout: 5 116712 8.28 MB/sec execute 171 sec latency 101.432 ms 2024-06-01T02:29:10.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:09 smithi027 ceph-mon[27271]: pgmap v381: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.0 MiB/s wr, 293 op/s 2024-06-01T02:29:11.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 116902 8.26 MB/sec execute 172 sec latency 98.868 ms 2024-06-01T02:29:12.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 117261 8.29 MB/sec execute 173 sec latency 58.384 ms 2024-06-01T02:29:12.717 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:12 smithi003 ceph-mon[25122]: pgmap v382: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.2 MiB/s wr, 448 op/s 2024-06-01T02:29:12.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:12 smithi027 ceph-mon[27271]: pgmap v382: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.2 MiB/s wr, 448 op/s 2024-06-01T02:29:13.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 117572 8.28 MB/sec execute 174 sec latency 51.286 ms 2024-06-01T02:29:14.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 117782 8.28 MB/sec execute 175 sec latency 150.139 ms 2024-06-01T02:29:14.540 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:14 smithi003 ceph-mon[25122]: pgmap v383: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 361 op/s 2024-06-01T02:29:14.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:14 smithi027 ceph-mon[27271]: pgmap v383: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 361 op/s 2024-06-01T02:29:15.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 118005 8.24 MB/sec execute 176 sec latency 76.456 ms 2024-06-01T02:29:16.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 118244 8.22 MB/sec execute 177 sec latency 65.772 ms 2024-06-01T02:29:16.606 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:16 smithi003 ceph-mon[25122]: pgmap v384: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.0 MiB/s wr, 368 op/s 2024-06-01T02:29:16.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:16 smithi027 ceph-mon[27271]: pgmap v384: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.0 MiB/s wr, 368 op/s 2024-06-01T02:29:17.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 118498 8.20 MB/sec execute 178 sec latency 83.677 ms 2024-06-01T02:29:18.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 118711 8.18 MB/sec execute 179 sec latency 225.829 ms 2024-06-01T02:29:18.658 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:18 smithi003 ceph-mon[25122]: pgmap v385: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.7 MiB/s wr, 510 op/s 2024-06-01T02:29:18.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:18 smithi027 ceph-mon[27271]: pgmap v385: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.7 MiB/s wr, 510 op/s 2024-06-01T02:29:19.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 119002 8.18 MB/sec execute 180 sec latency 122.945 ms 2024-06-01T02:29:19.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:19 smithi003 ceph-mon[25122]: pgmap v386: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-06-01T02:29:20.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 119347 8.17 MB/sec execute 181 sec latency 68.135 ms 2024-06-01T02:29:20.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:19 smithi027 ceph-mon[27271]: pgmap v386: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-06-01T02:29:21.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 119680 8.20 MB/sec execute 182 sec latency 67.052 ms 2024-06-01T02:29:22.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 119962 8.19 MB/sec execute 183 sec latency 69.811 ms 2024-06-01T02:29:22.513 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:22 smithi003 ceph-mon[25122]: pgmap v387: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.5 MiB/s wr, 489 op/s 2024-06-01T02:29:22.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:22 smithi027 ceph-mon[27271]: pgmap v387: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.5 MiB/s wr, 489 op/s 2024-06-01T02:29:23.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 120294 8.22 MB/sec execute 184 sec latency 41.975 ms 2024-06-01T02:29:24.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 120548 8.21 MB/sec execute 185 sec latency 81.920 ms 2024-06-01T02:29:24.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:24 smithi003 ceph-mon[25122]: pgmap v388: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 369 op/s 2024-06-01T02:29:24.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:24 smithi027 ceph-mon[27271]: pgmap v388: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 369 op/s 2024-06-01T02:29:25.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 120838 8.23 MB/sec execute 186 sec latency 86.597 ms 2024-06-01T02:29:25.570 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:29:25.570 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:29:25.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:29:25.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:29:26.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 121086 8.21 MB/sec execute 187 sec latency 143.977 ms 2024-06-01T02:29:26.694 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:26 smithi003 ceph-mon[25122]: pgmap v389: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 368 op/s 2024-06-01T02:29:26.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:26 smithi027 ceph-mon[27271]: pgmap v389: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 368 op/s 2024-06-01T02:29:27.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 121426 8.21 MB/sec execute 188 sec latency 61.298 ms 2024-06-01T02:29:28.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 121741 8.19 MB/sec execute 189 sec latency 143.874 ms 2024-06-01T02:29:28.753 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:28 smithi003 ceph-mon[25122]: pgmap v390: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 512 op/s 2024-06-01T02:29:28.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:28 smithi027 ceph-mon[27271]: pgmap v390: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 512 op/s 2024-06-01T02:29:29.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 121992 8.19 MB/sec execute 190 sec latency 83.234 ms 2024-06-01T02:29:30.059 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:29 smithi003 ceph-mon[25122]: pgmap v391: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 351 op/s 2024-06-01T02:29:30.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 122320 8.18 MB/sec execute 191 sec latency 55.982 ms 2024-06-01T02:29:30.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:29 smithi027 ceph-mon[27271]: pgmap v391: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 351 op/s 2024-06-01T02:29:31.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 122622 8.18 MB/sec execute 192 sec latency 125.527 ms 2024-06-01T02:29:32.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 122929 8.17 MB/sec execute 193 sec latency 131.222 ms 2024-06-01T02:29:32.670 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:32 smithi003 ceph-mon[25122]: pgmap v392: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.4 MiB/s wr, 422 op/s 2024-06-01T02:29:32.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:32 smithi027 ceph-mon[27271]: pgmap v392: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.4 MiB/s wr, 422 op/s 2024-06-01T02:29:33.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 123184 8.18 MB/sec execute 194 sec latency 130.443 ms 2024-06-01T02:29:34.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 123589 8.21 MB/sec execute 195 sec latency 52.271 ms 2024-06-01T02:29:34.676 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:34 smithi003 ceph-mon[25122]: pgmap v393: 97 pgs: 97 active+clean; 325 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.3 MiB/s wr, 380 op/s 2024-06-01T02:29:34.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:34 smithi027 ceph-mon[27271]: pgmap v393: 97 pgs: 97 active+clean; 325 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.3 MiB/s wr, 380 op/s 2024-06-01T02:29:35.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 123998 8.22 MB/sec execute 196 sec latency 55.898 ms 2024-06-01T02:29:36.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 124280 8.23 MB/sec execute 197 sec latency 41.869 ms 2024-06-01T02:29:36.721 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:36 smithi003 ceph-mon[25122]: pgmap v394: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.8 MiB/s wr, 490 op/s 2024-06-01T02:29:36.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:36 smithi027 ceph-mon[27271]: pgmap v394: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.8 MiB/s wr, 490 op/s 2024-06-01T02:29:37.276 INFO:teuthology.orchestra.run.smithi003.stdout: 5 124474 8.23 MB/sec execute 198 sec latency 95.857 ms 2024-06-01T02:29:38.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 124736 8.22 MB/sec execute 199 sec latency 80.115 ms 2024-06-01T02:29:38.539 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:38 smithi003 ceph-mon[25122]: pgmap v395: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 498 op/s 2024-06-01T02:29:38.539 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:38 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:29:38.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:38 smithi027 ceph-mon[27271]: pgmap v395: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 498 op/s 2024-06-01T02:29:38.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:38 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:29:39.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 125041 8.20 MB/sec execute 200 sec latency 57.057 ms 2024-06-01T02:29:40.101 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:39 smithi003 ceph-mon[25122]: pgmap v396: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 332 op/s 2024-06-01T02:29:40.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 125321 8.18 MB/sec execute 201 sec latency 50.817 ms 2024-06-01T02:29:40.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:39 smithi027 ceph-mon[27271]: pgmap v396: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 332 op/s 2024-06-01T02:29:41.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 125561 8.18 MB/sec execute 202 sec latency 86.187 ms 2024-06-01T02:29:42.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 125989 8.19 MB/sec execute 203 sec latency 54.258 ms 2024-06-01T02:29:42.647 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:42 smithi003 ceph-mon[25122]: pgmap v397: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.8 MiB/s wr, 475 op/s 2024-06-01T02:29:42.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:42 smithi027 ceph-mon[27271]: pgmap v397: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.8 MiB/s wr, 475 op/s 2024-06-01T02:29:43.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 126369 8.18 MB/sec execute 204 sec latency 51.578 ms 2024-06-01T02:29:44.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 126638 8.18 MB/sec execute 205 sec latency 82.088 ms 2024-06-01T02:29:44.714 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:44 smithi003 ceph-mon[25122]: pgmap v398: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.6 MiB/s wr, 482 op/s 2024-06-01T02:29:44.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:44 smithi027 ceph-mon[27271]: pgmap v398: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.6 MiB/s wr, 482 op/s 2024-06-01T02:29:45.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 126885 8.19 MB/sec execute 206 sec latency 90.779 ms 2024-06-01T02:29:46.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 127207 8.20 MB/sec execute 207 sec latency 57.734 ms 2024-06-01T02:29:46.536 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:46 smithi003 ceph-mon[25122]: pgmap v399: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 441 op/s 2024-06-01T02:29:46.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:46 smithi027 ceph-mon[27271]: pgmap v399: 97 pgs: 97 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 441 op/s 2024-06-01T02:29:47.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 127517 8.20 MB/sec execute 208 sec latency 52.235 ms 2024-06-01T02:29:48.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 127696 8.19 MB/sec execute 209 sec latency 93.939 ms 2024-06-01T02:29:48.578 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:48 smithi003 ceph-mon[25122]: pgmap v400: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 444 op/s 2024-06-01T02:29:48.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:48 smithi027 ceph-mon[27271]: pgmap v400: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 444 op/s 2024-06-01T02:29:49.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 127978 8.21 MB/sec execute 210 sec latency 78.749 ms 2024-06-01T02:29:50.145 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:49 smithi003 ceph-mon[25122]: pgmap v401: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.7 MiB/s wr, 406 op/s 2024-06-01T02:29:50.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 128263 8.20 MB/sec execute 211 sec latency 69.485 ms 2024-06-01T02:29:50.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:49 smithi027 ceph-mon[27271]: pgmap v401: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.7 MiB/s wr, 406 op/s 2024-06-01T02:29:51.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 128582 8.17 MB/sec execute 212 sec latency 126.255 ms 2024-06-01T02:29:52.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 128936 8.19 MB/sec execute 213 sec latency 49.984 ms 2024-06-01T02:29:52.696 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:52 smithi003 ceph-mon[25122]: pgmap v402: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.7 MiB/s wr, 488 op/s 2024-06-01T02:29:52.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:52 smithi027 ceph-mon[27271]: pgmap v402: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.7 MiB/s wr, 488 op/s 2024-06-01T02:29:53.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 129182 8.17 MB/sec execute 214 sec latency 64.442 ms 2024-06-01T02:29:54.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 129601 8.19 MB/sec execute 215 sec latency 71.779 ms 2024-06-01T02:29:54.716 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:54 smithi003 ceph-mon[25122]: pgmap v403: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 391 op/s 2024-06-01T02:29:54.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:54 smithi027 ceph-mon[27271]: pgmap v403: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 391 op/s 2024-06-01T02:29:55.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 129998 8.18 MB/sec execute 216 sec latency 86.415 ms 2024-06-01T02:29:56.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 130280 8.19 MB/sec execute 217 sec latency 39.322 ms 2024-06-01T02:29:56.553 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:56 smithi003 ceph-mon[25122]: pgmap v404: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 464 op/s 2024-06-01T02:29:56.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:56 smithi027 ceph-mon[27271]: pgmap v404: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 464 op/s 2024-06-01T02:29:57.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 130557 8.21 MB/sec execute 218 sec latency 129.599 ms 2024-06-01T02:29:58.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 130927 8.21 MB/sec execute 219 sec latency 86.009 ms 2024-06-01T02:29:58.618 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:58 smithi003 ceph-mon[25122]: pgmap v405: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.8 MiB/s wr, 525 op/s 2024-06-01T02:29:58.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:58 smithi027 ceph-mon[27271]: pgmap v405: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.8 MiB/s wr, 525 op/s 2024-06-01T02:29:59.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 131203 8.22 MB/sec execute 220 sec latency 134.381 ms 2024-06-01T02:30:00.195 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:29:59 smithi003 ceph-mon[25122]: pgmap v406: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 375 op/s 2024-06-01T02:30:00.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 131418 8.21 MB/sec execute 221 sec latency 77.295 ms 2024-06-01T02:30:00.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:29:59 smithi027 ceph-mon[27271]: pgmap v406: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 375 op/s 2024-06-01T02:30:00.960 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: Health detail: HEALTH_WARN failed to probe daemons or devices 2024-06-01T02:30:00.960 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: [WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices 2024-06-01T02:30:00.960 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: host smithi003 `cephadm gather-facts` failed: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /bin/podman version --format {{.Client.Version}} 2024-06-01T02:30:00.960 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: /bin/podman: stderr Error: beginning transaction: database is locked 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: Traceback (most recent call last): 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8634, in 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: main() 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8620, in main 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: check_container_engine(ctx) 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 2165, in check_container_engine 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: engine.get_version(ctx) 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 195, in get_version 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}']) 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 1615, in call_throws 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: raise RuntimeError(f'Failed command: {" ".join(command)}: {s}') 2024-06-01T02:30:00.961 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:00 smithi003 ceph-mon[25122]: RuntimeError: Failed command: /bin/podman version --format {{.Client.Version}}: Error: beginning transaction: database is locked 2024-06-01T02:30:01.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 131605 8.20 MB/sec execute 222 sec latency 80.270 ms 2024-06-01T02:30:01.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: Health detail: HEALTH_WARN failed to probe daemons or devices 2024-06-01T02:30:01.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: [WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices 2024-06-01T02:30:01.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: host smithi003 `cephadm gather-facts` failed: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /bin/podman version --format {{.Client.Version}} 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: /bin/podman: stderr Error: beginning transaction: database is locked 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: Traceback (most recent call last): 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8634, in 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: main() 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8620, in main 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: check_container_engine(ctx) 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 2165, in check_container_engine 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: engine.get_version(ctx) 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 195, in get_version 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}']) 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 1615, in call_throws 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: raise RuntimeError(f'Failed command: {" ".join(command)}: {s}') 2024-06-01T02:30:01.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:00 smithi027 ceph-mon[27271]: RuntimeError: Failed command: /bin/podman version --format {{.Client.Version}}: Error: beginning transaction: database is locked 2024-06-01T02:30:02.000 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:01 smithi003 ceph-mon[25122]: pgmap v407: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 460 op/s 2024-06-01T02:30:02.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 131874 8.19 MB/sec execute 223 sec latency 94.085 ms 2024-06-01T02:30:02.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:01 smithi027 ceph-mon[27271]: pgmap v407: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 460 op/s 2024-06-01T02:30:03.277 INFO:teuthology.orchestra.run.smithi003.stdout: 5 132198 8.18 MB/sec execute 224 sec latency 137.366 ms 2024-06-01T02:30:04.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 132421 8.16 MB/sec execute 225 sec latency 91.169 ms 2024-06-01T02:30:04.568 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:04 smithi003 ceph-mon[25122]: pgmap v408: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 437 op/s 2024-06-01T02:30:04.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:04 smithi027 ceph-mon[27271]: pgmap v408: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 437 op/s 2024-06-01T02:30:05.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 132653 8.16 MB/sec execute 226 sec latency 112.771 ms 2024-06-01T02:30:06.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 133000 8.17 MB/sec execute 227 sec latency 96.964 ms 2024-06-01T02:30:06.652 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:06 smithi003 ceph-mon[25122]: pgmap v409: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 477 op/s 2024-06-01T02:30:06.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:06 smithi027 ceph-mon[27271]: pgmap v409: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 477 op/s 2024-06-01T02:30:07.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 133346 8.15 MB/sec execute 228 sec latency 73.386 ms 2024-06-01T02:30:08.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 133649 8.15 MB/sec execute 229 sec latency 79.810 ms 2024-06-01T02:30:08.696 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:08 smithi003 ceph-mon[25122]: pgmap v410: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 407 op/s 2024-06-01T02:30:08.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:08 smithi027 ceph-mon[27271]: pgmap v410: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 407 op/s 2024-06-01T02:30:09.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 133869 8.15 MB/sec execute 230 sec latency 69.495 ms 2024-06-01T02:30:09.993 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:09 smithi003 ceph-mon[25122]: pgmap v411: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.4 MiB/s wr, 312 op/s 2024-06-01T02:30:10.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 134162 8.16 MB/sec execute 231 sec latency 84.111 ms 2024-06-01T02:30:10.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:09 smithi027 ceph-mon[27271]: pgmap v411: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.4 MiB/s wr, 312 op/s 2024-06-01T02:30:11.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 134472 8.16 MB/sec execute 232 sec latency 107.711 ms 2024-06-01T02:30:12.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 134804 8.17 MB/sec execute 233 sec latency 49.102 ms 2024-06-01T02:30:12.508 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:12 smithi003 ceph-mon[25122]: pgmap v412: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.2 MiB/s wr, 375 op/s 2024-06-01T02:30:12.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:12 smithi027 ceph-mon[27271]: pgmap v412: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.2 MiB/s wr, 375 op/s 2024-06-01T02:30:13.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 135051 8.18 MB/sec execute 234 sec latency 79.712 ms 2024-06-01T02:30:14.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 135286 8.17 MB/sec execute 235 sec latency 78.218 ms 2024-06-01T02:30:14.583 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:14 smithi003 ceph-mon[25122]: pgmap v413: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-06-01T02:30:14.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:14 smithi027 ceph-mon[27271]: pgmap v413: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-06-01T02:30:15.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 135535 8.15 MB/sec execute 236 sec latency 125.507 ms 2024-06-01T02:30:16.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 135839 8.14 MB/sec execute 237 sec latency 77.508 ms 2024-06-01T02:30:16.617 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:16 smithi003 ceph-mon[25122]: pgmap v414: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-06-01T02:30:16.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:16 smithi027 ceph-mon[27271]: pgmap v414: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-06-01T02:30:17.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 135998 8.12 MB/sec execute 238 sec latency 79.743 ms 2024-06-01T02:30:18.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 136222 8.12 MB/sec execute 239 sec latency 70.778 ms 2024-06-01T02:30:18.696 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:18 smithi003 ceph-mon[25122]: pgmap v415: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.4 MiB/s wr, 445 op/s 2024-06-01T02:30:18.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:18 smithi027 ceph-mon[27271]: pgmap v415: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.4 MiB/s wr, 445 op/s 2024-06-01T02:30:19.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 136564 8.13 MB/sec execute 240 sec latency 56.419 ms 2024-06-01T02:30:20.178 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:19 smithi003 ceph-mon[25122]: pgmap v416: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.6 MiB/s wr, 362 op/s 2024-06-01T02:30:20.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 136950 8.11 MB/sec execute 241 sec latency 54.090 ms 2024-06-01T02:30:20.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:19 smithi027 ceph-mon[27271]: pgmap v416: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.6 MiB/s wr, 362 op/s 2024-06-01T02:30:21.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 137187 8.10 MB/sec execute 242 sec latency 99.267 ms 2024-06-01T02:30:22.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 137375 8.09 MB/sec execute 243 sec latency 59.560 ms 2024-06-01T02:30:22.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:22 smithi003 ceph-mon[25122]: pgmap v417: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 454 op/s 2024-06-01T02:30:22.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:22 smithi027 ceph-mon[27271]: pgmap v417: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 454 op/s 2024-06-01T02:30:23.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 137806 8.13 MB/sec execute 244 sec latency 58.184 ms 2024-06-01T02:30:24.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 138222 8.14 MB/sec execute 245 sec latency 89.960 ms 2024-06-01T02:30:24.627 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:24 smithi003 ceph-mon[25122]: pgmap v418: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 448 op/s 2024-06-01T02:30:24.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:24 smithi027 ceph-mon[27271]: pgmap v418: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 448 op/s 2024-06-01T02:30:25.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 138493 8.15 MB/sec execute 246 sec latency 71.009 ms 2024-06-01T02:30:25.654 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:30:25.654 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:30:25.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:30:25.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:30:26.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 138794 8.16 MB/sec execute 247 sec latency 81.671 ms 2024-06-01T02:30:26.667 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:26 smithi003 ceph-mon[25122]: pgmap v419: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 341 op/s 2024-06-01T02:30:26.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:26 smithi027 ceph-mon[27271]: pgmap v419: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 341 op/s 2024-06-01T02:30:27.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 139123 8.14 MB/sec execute 248 sec latency 64.648 ms 2024-06-01T02:30:28.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 139395 8.13 MB/sec execute 249 sec latency 155.048 ms 2024-06-01T02:30:28.764 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:28 smithi003 ceph-mon[25122]: pgmap v420: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 476 op/s 2024-06-01T02:30:28.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:28 smithi027 ceph-mon[27271]: pgmap v420: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 476 op/s 2024-06-01T02:30:29.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 139603 8.12 MB/sec execute 250 sec latency 82.823 ms 2024-06-01T02:30:30.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:29 smithi003 ceph-mon[25122]: pgmap v421: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 340 op/s 2024-06-01T02:30:30.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 139938 8.15 MB/sec execute 251 sec latency 67.777 ms 2024-06-01T02:30:30.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:29 smithi027 ceph-mon[27271]: pgmap v421: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 340 op/s 2024-06-01T02:30:31.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 140363 8.13 MB/sec execute 252 sec latency 69.104 ms 2024-06-01T02:30:32.278 INFO:teuthology.orchestra.run.smithi003.stdout: 5 140737 8.13 MB/sec execute 253 sec latency 87.389 ms 2024-06-01T02:30:32.632 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:32 smithi003 ceph-mon[25122]: pgmap v422: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 499 op/s 2024-06-01T02:30:32.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:32 smithi027 ceph-mon[27271]: pgmap v422: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 499 op/s 2024-06-01T02:30:33.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 140918 8.12 MB/sec execute 254 sec latency 140.607 ms 2024-06-01T02:30:34.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 141382 8.15 MB/sec execute 255 sec latency 65.061 ms 2024-06-01T02:30:34.669 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:34 smithi003 ceph-mon[25122]: pgmap v423: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-06-01T02:30:34.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:34 smithi027 ceph-mon[27271]: pgmap v423: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-06-01T02:30:35.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 141693 8.15 MB/sec execute 256 sec latency 116.025 ms 2024-06-01T02:30:36.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 141996 8.17 MB/sec execute 257 sec latency 62.111 ms 2024-06-01T02:30:36.723 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:36 smithi003 ceph-mon[25122]: pgmap v424: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 414 op/s 2024-06-01T02:30:36.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:36 smithi027 ceph-mon[27271]: pgmap v424: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 414 op/s 2024-06-01T02:30:37.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 142184 8.16 MB/sec execute 258 sec latency 72.407 ms 2024-06-01T02:30:38.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 142522 8.16 MB/sec execute 259 sec latency 155.818 ms 2024-06-01T02:30:38.755 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:38 smithi003 ceph-mon[25122]: pgmap v425: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 560 op/s 2024-06-01T02:30:38.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:38 smithi027 ceph-mon[27271]: pgmap v425: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 560 op/s 2024-06-01T02:30:39.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 142847 8.14 MB/sec execute 260 sec latency 140.482 ms 2024-06-01T02:30:40.013 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:39 smithi003 ceph-mon[25122]: pgmap v426: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 401 op/s 2024-06-01T02:30:40.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 143110 8.14 MB/sec execute 261 sec latency 83.664 ms 2024-06-01T02:30:40.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:39 smithi027 ceph-mon[27271]: pgmap v426: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 401 op/s 2024-06-01T02:30:41.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 143477 8.17 MB/sec execute 262 sec latency 104.149 ms 2024-06-01T02:30:42.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 143917 8.15 MB/sec execute 263 sec latency 62.898 ms 2024-06-01T02:30:42.562 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:42 smithi003 ceph-mon[25122]: pgmap v427: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 538 op/s 2024-06-01T02:30:42.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:42 smithi027 ceph-mon[27271]: pgmap v427: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 538 op/s 2024-06-01T02:30:43.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 144239 8.15 MB/sec execute 264 sec latency 97.333 ms 2024-06-01T02:30:44.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 144470 8.15 MB/sec execute 265 sec latency 69.271 ms 2024-06-01T02:30:44.629 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:44 smithi003 ceph-mon[25122]: pgmap v428: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 415 op/s 2024-06-01T02:30:44.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:44 smithi027 ceph-mon[27271]: pgmap v428: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 415 op/s 2024-06-01T02:30:45.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 144779 8.17 MB/sec execute 266 sec latency 52.658 ms 2024-06-01T02:30:46.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 145132 8.16 MB/sec execute 267 sec latency 52.312 ms 2024-06-01T02:30:46.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:46 smithi003 ceph-mon[25122]: pgmap v429: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.6 MiB/s wr, 417 op/s 2024-06-01T02:30:46.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:46 smithi027 ceph-mon[27271]: pgmap v429: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.6 MiB/s wr, 417 op/s 2024-06-01T02:30:47.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 145497 8.19 MB/sec execute 268 sec latency 94.011 ms 2024-06-01T02:30:48.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 145757 8.19 MB/sec execute 269 sec latency 98.694 ms 2024-06-01T02:30:48.750 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:48 smithi003 ceph-mon[25122]: pgmap v430: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.4 MiB/s wr, 543 op/s 2024-06-01T02:30:48.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:48 smithi027 ceph-mon[27271]: pgmap v430: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.4 MiB/s wr, 543 op/s 2024-06-01T02:30:49.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 145968 8.17 MB/sec execute 270 sec latency 73.691 ms 2024-06-01T02:30:49.959 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:49 smithi003 ceph-mon[25122]: pgmap v431: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 364 op/s 2024-06-01T02:30:50.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 146315 8.16 MB/sec execute 271 sec latency 96.977 ms 2024-06-01T02:30:50.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:49 smithi027 ceph-mon[27271]: pgmap v431: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 364 op/s 2024-06-01T02:30:51.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 146515 8.15 MB/sec execute 272 sec latency 51.382 ms 2024-06-01T02:30:52.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 146807 8.15 MB/sec execute 273 sec latency 47.687 ms 2024-06-01T02:30:52.545 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:52 smithi003 ceph-mon[25122]: pgmap v432: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.0 MiB/s wr, 507 op/s 2024-06-01T02:30:52.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:52 smithi027 ceph-mon[27271]: pgmap v432: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.0 MiB/s wr, 507 op/s 2024-06-01T02:30:53.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 147217 8.17 MB/sec execute 274 sec latency 54.082 ms 2024-06-01T02:30:54.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 147683 8.16 MB/sec execute 275 sec latency 93.895 ms 2024-06-01T02:30:54.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:54 smithi003 ceph-mon[25122]: pgmap v433: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 424 op/s 2024-06-01T02:30:54.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:54 smithi027 ceph-mon[27271]: pgmap v433: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 424 op/s 2024-06-01T02:30:55.279 INFO:teuthology.orchestra.run.smithi003.stdout: 5 147838 8.15 MB/sec execute 276 sec latency 71.863 ms 2024-06-01T02:30:56.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 147936 8.13 MB/sec execute 277 sec latency 95.625 ms 2024-06-01T02:30:56.659 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:56 smithi003 ceph-mon[25122]: pgmap v434: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.6 MiB/s wr, 418 op/s 2024-06-01T02:30:56.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:56 smithi027 ceph-mon[27271]: pgmap v434: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.6 MiB/s wr, 418 op/s 2024-06-01T02:30:57.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 148101 8.13 MB/sec execute 278 sec latency 101.751 ms 2024-06-01T02:30:58.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 148410 8.13 MB/sec execute 279 sec latency 62.408 ms 2024-06-01T02:30:58.777 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:58 smithi003 ceph-mon[25122]: pgmap v435: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 490 op/s 2024-06-01T02:30:58.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:58 smithi027 ceph-mon[27271]: pgmap v435: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 490 op/s 2024-06-01T02:30:59.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 148717 8.13 MB/sec execute 280 sec latency 87.888 ms 2024-06-01T02:31:00.052 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:30:59 smithi003 ceph-mon[25122]: pgmap v436: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.3 MiB/s wr, 332 op/s 2024-06-01T02:31:00.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 148994 8.14 MB/sec execute 281 sec latency 115.875 ms 2024-06-01T02:31:00.349 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:30:59 smithi027 ceph-mon[27271]: pgmap v436: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.3 MiB/s wr, 332 op/s 2024-06-01T02:31:01.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 149159 8.13 MB/sec execute 282 sec latency 67.562 ms 2024-06-01T02:31:02.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 149385 8.13 MB/sec execute 283 sec latency 75.392 ms 2024-06-01T02:31:02.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:02 smithi003 ceph-mon[25122]: pgmap v437: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 455 op/s 2024-06-01T02:31:02.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:02 smithi027 ceph-mon[27271]: pgmap v437: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 455 op/s 2024-06-01T02:31:03.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 149518 8.11 MB/sec execute 284 sec latency 77.946 ms 2024-06-01T02:31:04.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 149764 8.09 MB/sec execute 285 sec latency 78.720 ms 2024-06-01T02:31:04.720 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:04 smithi003 ceph-mon[25122]: pgmap v438: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 331 op/s 2024-06-01T02:31:04.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:04 smithi027 ceph-mon[27271]: pgmap v438: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 331 op/s 2024-06-01T02:31:05.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 149984 8.08 MB/sec execute 286 sec latency 103.527 ms 2024-06-01T02:31:06.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 150157 8.07 MB/sec execute 287 sec latency 82.530 ms 2024-06-01T02:31:06.534 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:06 smithi003 ceph-mon[25122]: pgmap v439: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.8 MiB/s wr, 382 op/s 2024-06-01T02:31:06.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:06 smithi027 ceph-mon[27271]: pgmap v439: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.8 MiB/s wr, 382 op/s 2024-06-01T02:31:07.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 150434 8.08 MB/sec execute 288 sec latency 103.878 ms 2024-06-01T02:31:08.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 150691 8.06 MB/sec execute 289 sec latency 118.414 ms 2024-06-01T02:31:08.604 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:08 smithi003 ceph-mon[25122]: pgmap v440: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 381 op/s 2024-06-01T02:31:08.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:08 smithi027 ceph-mon[27271]: pgmap v440: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 381 op/s 2024-06-01T02:31:09.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 151120 8.07 MB/sec execute 290 sec latency 148.507 ms 2024-06-01T02:31:10.158 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:09 smithi003 ceph-mon[25122]: pgmap v441: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.9 MiB/s wr, 275 op/s 2024-06-01T02:31:10.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 151393 8.05 MB/sec execute 291 sec latency 77.438 ms 2024-06-01T02:31:10.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:09 smithi027 ceph-mon[27271]: pgmap v441: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.9 MiB/s wr, 275 op/s 2024-06-01T02:31:11.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 151604 8.04 MB/sec execute 292 sec latency 151.141 ms 2024-06-01T02:31:12.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 151874 8.06 MB/sec execute 293 sec latency 103.699 ms 2024-06-01T02:31:12.709 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:12 smithi003 ceph-mon[25122]: pgmap v442: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.2 MiB/s wr, 444 op/s 2024-06-01T02:31:12.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:12 smithi027 ceph-mon[27271]: pgmap v442: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.2 MiB/s wr, 444 op/s 2024-06-01T02:31:13.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 152315 8.07 MB/sec execute 294 sec latency 69.394 ms 2024-06-01T02:31:14.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 152840 8.11 MB/sec execute 295 sec latency 63.954 ms 2024-06-01T02:31:14.735 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:14 smithi003 ceph-mon[25122]: pgmap v443: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.5 MiB/s wr, 347 op/s 2024-06-01T02:31:14.849 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:14 smithi027 ceph-mon[27271]: pgmap v443: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.5 MiB/s wr, 347 op/s 2024-06-01T02:31:15.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 153317 8.11 MB/sec execute 296 sec latency 43.533 ms 2024-06-01T02:31:16.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 153624 8.12 MB/sec execute 297 sec latency 94.245 ms 2024-06-01T02:31:16.773 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:16 smithi003 ceph-mon[25122]: pgmap v444: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 417 op/s 2024-06-01T02:31:16.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:16 smithi027 ceph-mon[27271]: pgmap v444: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 417 op/s 2024-06-01T02:31:17.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 153803 8.11 MB/sec execute 298 sec latency 141.577 ms 2024-06-01T02:31:18.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 154111 8.11 MB/sec execute 299 sec latency 92.665 ms 2024-06-01T02:31:18.561 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:18 smithi003 ceph-mon[25122]: pgmap v445: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.7 MiB/s wr, 409 op/s 2024-06-01T02:31:18.561 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:31:18.561 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:31:18.561 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:31:18.561 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:31:18.561 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:31:18.561 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:31:18.561 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:31:18.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:18 smithi027 ceph-mon[27271]: pgmap v445: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.7 MiB/s wr, 409 op/s 2024-06-01T02:31:18.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:31:18.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:31:18.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:31:18.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:31:18.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:31:18.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:31:18.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:31:19.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 154415 8.10 MB/sec execute 300 sec latency 100.649 ms 2024-06-01T02:31:19.581 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:19 smithi003 ceph-mon[25122]: Adjusting osd_memory_target on smithi003 to 1937M 2024-06-01T02:31:19.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:19 smithi003 ceph-mon[25122]: pgmap v446: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 425 op/s 2024-06-01T02:31:19.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:19 smithi003 ceph-mon[25122]: pgmap v447: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.6 MiB/s wr, 522 op/s 2024-06-01T02:31:19.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:19 smithi003 ceph-mon[25122]: pgmap v448: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 377 op/s 2024-06-01T02:31:19.582 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:19 smithi003 ceph-mon[25122]: Health check failed: 1 hosts fail cephadm check (CEPHADM_HOST_CHECK_FAILED) 2024-06-01T02:31:19.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:19 smithi027 ceph-mon[27271]: Adjusting osd_memory_target on smithi003 to 1937M 2024-06-01T02:31:19.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:19 smithi027 ceph-mon[27271]: pgmap v446: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 425 op/s 2024-06-01T02:31:19.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:19 smithi027 ceph-mon[27271]: pgmap v447: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.6 MiB/s wr, 522 op/s 2024-06-01T02:31:19.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:19 smithi027 ceph-mon[27271]: pgmap v448: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 377 op/s 2024-06-01T02:31:19.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:19 smithi027 ceph-mon[27271]: Health check failed: 1 hosts fail cephadm check (CEPHADM_HOST_CHECK_FAILED) 2024-06-01T02:31:20.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 154726 8.10 MB/sec execute 301 sec latency 70.621 ms 2024-06-01T02:31:21.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 155019 8.09 MB/sec execute 302 sec latency 66.782 ms 2024-06-01T02:31:21.628 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:21 smithi003 ceph-mon[25122]: pgmap v449: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 535 op/s 2024-06-01T02:31:21.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:21 smithi027 ceph-mon[27271]: pgmap v449: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 535 op/s 2024-06-01T02:31:22.280 INFO:teuthology.orchestra.run.smithi003.stdout: 5 155306 8.08 MB/sec execute 303 sec latency 55.591 ms 2024-06-01T02:31:23.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 155573 8.09 MB/sec execute 304 sec latency 110.009 ms 2024-06-01T02:31:23.687 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:23 smithi003 ceph-mon[25122]: pgmap v450: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-06-01T02:31:23.850 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:23 smithi027 ceph-mon[27271]: pgmap v450: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-06-01T02:31:24.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 155886 8.10 MB/sec execute 305 sec latency 69.074 ms 2024-06-01T02:31:24.987 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:24 smithi003 ceph-mon[25122]: pgmap v451: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.2 MiB/s wr, 453 op/s 2024-06-01T02:31:24.988 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:31:24.988 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:31:24.988 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:31:25.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 156161 8.11 MB/sec execute 306 sec latency 83.546 ms 2024-06-01T02:31:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:24 smithi027 ceph-mon[27271]: pgmap v451: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.2 MiB/s wr, 453 op/s 2024-06-01T02:31:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:31:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:31:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:31:26.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 156332 8.09 MB/sec execute 307 sec latency 92.896 ms 2024-06-01T02:31:27.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 156555 8.08 MB/sec execute 308 sec latency 65.392 ms 2024-06-01T02:31:27.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:27 smithi003 ceph-mon[25122]: pgmap v452: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.3 MiB/s wr, 507 op/s 2024-06-01T02:31:27.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:27 smithi027 ceph-mon[27271]: pgmap v452: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.3 MiB/s wr, 507 op/s 2024-06-01T02:31:28.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 156804 8.08 MB/sec execute 309 sec latency 51.553 ms 2024-06-01T02:31:29.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 157118 8.09 MB/sec execute 310 sec latency 57.206 ms 2024-06-01T02:31:29.409 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:29 smithi003 ceph-mon[25122]: pgmap v453: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.0 MiB/s wr, 441 op/s 2024-06-01T02:31:29.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:29 smithi027 ceph-mon[27271]: pgmap v453: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.0 MiB/s wr, 441 op/s 2024-06-01T02:31:30.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 157403 8.09 MB/sec execute 311 sec latency 119.814 ms 2024-06-01T02:31:31.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 157668 8.08 MB/sec execute 312 sec latency 53.334 ms 2024-06-01T02:31:31.356 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:31 smithi003 ceph-mon[25122]: pgmap v454: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.4 MiB/s wr, 369 op/s 2024-06-01T02:31:31.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:31 smithi027 ceph-mon[27271]: pgmap v454: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.4 MiB/s wr, 369 op/s 2024-06-01T02:31:32.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 157961 8.08 MB/sec execute 313 sec latency 57.803 ms 2024-06-01T02:31:33.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 158288 8.07 MB/sec execute 314 sec latency 45.845 ms 2024-06-01T02:31:33.382 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:33 smithi003 ceph-mon[25122]: pgmap v455: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-06-01T02:31:33.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:33 smithi027 ceph-mon[27271]: pgmap v455: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-06-01T02:31:34.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 158611 8.06 MB/sec execute 315 sec latency 71.312 ms 2024-06-01T02:31:35.192 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:34 smithi003 ceph-mon[25122]: pgmap v456: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 381 op/s 2024-06-01T02:31:35.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 158894 8.07 MB/sec execute 316 sec latency 69.064 ms 2024-06-01T02:31:35.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:34 smithi027 ceph-mon[27271]: pgmap v456: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 381 op/s 2024-06-01T02:31:36.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 159201 8.07 MB/sec execute 317 sec latency 74.531 ms 2024-06-01T02:31:37.269 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:37 smithi003 ceph-mon[25122]: pgmap v457: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 425 op/s 2024-06-01T02:31:37.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 159548 8.08 MB/sec execute 318 sec latency 76.982 ms 2024-06-01T02:31:37.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:37 smithi027 ceph-mon[27271]: pgmap v457: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 425 op/s 2024-06-01T02:31:38.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 159879 8.08 MB/sec execute 319 sec latency 71.592 ms 2024-06-01T02:31:39.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 160167 8.09 MB/sec execute 320 sec latency 90.951 ms 2024-06-01T02:31:39.319 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:39 smithi003 ceph-mon[25122]: pgmap v458: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 422 op/s 2024-06-01T02:31:39.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:39 smithi027 ceph-mon[27271]: pgmap v458: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 422 op/s 2024-06-01T02:31:40.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 160408 8.08 MB/sec execute 321 sec latency 54.245 ms 2024-06-01T02:31:41.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 160709 8.09 MB/sec execute 322 sec latency 122.252 ms 2024-06-01T02:31:41.388 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:41 smithi003 ceph-mon[25122]: pgmap v459: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.8 MiB/s wr, 407 op/s 2024-06-01T02:31:41.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:41 smithi027 ceph-mon[27271]: pgmap v459: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.8 MiB/s wr, 407 op/s 2024-06-01T02:31:42.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 160986 8.08 MB/sec execute 323 sec latency 91.108 ms 2024-06-01T02:31:43.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 161267 8.08 MB/sec execute 324 sec latency 80.186 ms 2024-06-01T02:31:43.444 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:43 smithi003 ceph-mon[25122]: pgmap v460: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 500 op/s 2024-06-01T02:31:43.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:43 smithi027 ceph-mon[27271]: pgmap v460: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 500 op/s 2024-06-01T02:31:44.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 161544 8.07 MB/sec execute 325 sec latency 78.710 ms 2024-06-01T02:31:44.994 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:44 smithi003 ceph-mon[25122]: pgmap v461: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 337 op/s 2024-06-01T02:31:45.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 161842 8.06 MB/sec execute 326 sec latency 66.038 ms 2024-06-01T02:31:45.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:44 smithi027 ceph-mon[27271]: pgmap v461: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 337 op/s 2024-06-01T02:31:46.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 162160 8.06 MB/sec execute 327 sec latency 118.986 ms 2024-06-01T02:31:47.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 162417 8.06 MB/sec execute 328 sec latency 87.287 ms 2024-06-01T02:31:47.388 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:47 smithi003 ceph-mon[25122]: pgmap v462: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 520 op/s 2024-06-01T02:31:47.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:47 smithi027 ceph-mon[27271]: pgmap v462: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 520 op/s 2024-06-01T02:31:48.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 162665 8.05 MB/sec execute 329 sec latency 111.004 ms 2024-06-01T02:31:49.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 162961 8.06 MB/sec execute 330 sec latency 62.631 ms 2024-06-01T02:31:49.463 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:49 smithi003 ceph-mon[25122]: pgmap v463: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 433 op/s 2024-06-01T02:31:49.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:49 smithi027 ceph-mon[27271]: pgmap v463: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 433 op/s 2024-06-01T02:31:50.281 INFO:teuthology.orchestra.run.smithi003.stdout: 5 163223 8.06 MB/sec execute 331 sec latency 205.172 ms 2024-06-01T02:31:51.282 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:51 smithi003 ceph-mon[25122]: pgmap v464: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.6 MiB/s wr, 354 op/s 2024-06-01T02:31:51.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 163490 8.06 MB/sec execute 332 sec latency 82.065 ms 2024-06-01T02:31:51.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:51 smithi027 ceph-mon[27271]: pgmap v464: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.6 MiB/s wr, 354 op/s 2024-06-01T02:31:52.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 163764 8.06 MB/sec execute 333 sec latency 69.794 ms 2024-06-01T02:31:53.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 164041 8.07 MB/sec execute 334 sec latency 75.846 ms 2024-06-01T02:31:53.329 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:53 smithi003 ceph-mon[25122]: pgmap v465: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-06-01T02:31:53.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:53 smithi027 ceph-mon[27271]: pgmap v465: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-06-01T02:31:54.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 164345 8.07 MB/sec execute 335 sec latency 80.799 ms 2024-06-01T02:31:55.131 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:54 smithi003 ceph-mon[25122]: pgmap v466: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.5 MiB/s wr, 382 op/s 2024-06-01T02:31:55.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 164698 8.07 MB/sec execute 336 sec latency 85.863 ms 2024-06-01T02:31:55.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:54 smithi027 ceph-mon[27271]: pgmap v466: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.5 MiB/s wr, 382 op/s 2024-06-01T02:31:56.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 164934 8.06 MB/sec execute 337 sec latency 66.557 ms 2024-06-01T02:31:57.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 165170 8.05 MB/sec execute 338 sec latency 71.483 ms 2024-06-01T02:31:57.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:57 smithi003 ceph-mon[25122]: pgmap v467: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 530 op/s 2024-06-01T02:31:57.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:57 smithi027 ceph-mon[27271]: pgmap v467: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 530 op/s 2024-06-01T02:31:58.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 165535 8.05 MB/sec execute 339 sec latency 47.922 ms 2024-06-01T02:31:59.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 165853 8.04 MB/sec execute 340 sec latency 74.720 ms 2024-06-01T02:31:59.522 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:31:59 smithi003 ceph-mon[25122]: pgmap v468: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 366 op/s 2024-06-01T02:31:59.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:31:59 smithi027 ceph-mon[27271]: pgmap v468: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 366 op/s 2024-06-01T02:32:00.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 166055 8.05 MB/sec execute 341 sec latency 103.108 ms 2024-06-01T02:32:01.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 166237 8.03 MB/sec execute 342 sec latency 68.242 ms 2024-06-01T02:32:01.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:01 smithi003 ceph-mon[25122]: pgmap v469: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.5 MiB/s wr, 324 op/s 2024-06-01T02:32:01.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:01 smithi027 ceph-mon[27271]: pgmap v469: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.5 MiB/s wr, 324 op/s 2024-06-01T02:32:02.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 166609 8.04 MB/sec execute 343 sec latency 60.925 ms 2024-06-01T02:32:03.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 166915 8.04 MB/sec execute 344 sec latency 86.306 ms 2024-06-01T02:32:03.394 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:03 smithi003 ceph-mon[25122]: pgmap v470: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.0 MiB/s wr, 498 op/s 2024-06-01T02:32:03.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:03 smithi027 ceph-mon[27271]: pgmap v470: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.0 MiB/s wr, 498 op/s 2024-06-01T02:32:04.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 167128 8.03 MB/sec execute 345 sec latency 102.008 ms 2024-06-01T02:32:05.213 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:05 smithi003 ceph-mon[25122]: pgmap v471: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-06-01T02:32:05.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 167377 8.04 MB/sec execute 346 sec latency 87.407 ms 2024-06-01T02:32:05.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:05 smithi027 ceph-mon[27271]: pgmap v471: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-06-01T02:32:06.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 167612 8.04 MB/sec execute 347 sec latency 63.388 ms 2024-06-01T02:32:07.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 167899 8.05 MB/sec execute 348 sec latency 76.635 ms 2024-06-01T02:32:07.322 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:07 smithi003 ceph-mon[25122]: pgmap v472: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-06-01T02:32:07.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:07 smithi027 ceph-mon[27271]: pgmap v472: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-06-01T02:32:08.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 168120 8.04 MB/sec execute 349 sec latency 122.136 ms 2024-06-01T02:32:09.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 168313 8.02 MB/sec execute 350 sec latency 128.758 ms 2024-06-01T02:32:09.390 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:09 smithi003 ceph-mon[25122]: pgmap v473: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 350 op/s 2024-06-01T02:32:09.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:09 smithi027 ceph-mon[27271]: pgmap v473: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 350 op/s 2024-06-01T02:32:10.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 168501 8.01 MB/sec execute 351 sec latency 69.856 ms 2024-06-01T02:32:11.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 168694 8.00 MB/sec execute 352 sec latency 90.751 ms 2024-06-01T02:32:11.521 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:11 smithi003 ceph-mon[25122]: pgmap v474: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.4 MiB/s wr, 325 op/s 2024-06-01T02:32:11.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:11 smithi027 ceph-mon[27271]: pgmap v474: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.4 MiB/s wr, 325 op/s 2024-06-01T02:32:12.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 168965 8.00 MB/sec execute 353 sec latency 124.475 ms 2024-06-01T02:32:13.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 169189 7.98 MB/sec execute 354 sec latency 53.974 ms 2024-06-01T02:32:13.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:13 smithi003 ceph-mon[25122]: pgmap v475: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-06-01T02:32:13.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:13 smithi027 ceph-mon[27271]: pgmap v475: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-06-01T02:32:14.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 169410 7.97 MB/sec execute 355 sec latency 115.942 ms 2024-06-01T02:32:15.174 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:14 smithi003 ceph-mon[25122]: pgmap v476: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 283 op/s 2024-06-01T02:32:15.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 169631 7.98 MB/sec execute 356 sec latency 47.392 ms 2024-06-01T02:32:15.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:14 smithi027 ceph-mon[27271]: pgmap v476: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 283 op/s 2024-06-01T02:32:16.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 169862 7.98 MB/sec execute 357 sec latency 62.340 ms 2024-06-01T02:32:17.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 170082 7.96 MB/sec execute 358 sec latency 109.118 ms 2024-06-01T02:32:17.293 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:17 smithi003 ceph-mon[25122]: pgmap v477: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-06-01T02:32:17.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:17 smithi027 ceph-mon[27271]: pgmap v477: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-06-01T02:32:18.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 170364 7.96 MB/sec execute 359 sec latency 84.099 ms 2024-06-01T02:32:18.360 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:32:18.360 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:32:18.360 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:18 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:32:18.592 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:32:18.592 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:32:18.593 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:18 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:32:19.282 INFO:teuthology.orchestra.run.smithi003.stdout: 5 170596 7.96 MB/sec execute 360 sec latency 88.642 ms 2024-06-01T02:32:19.681 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:19 smithi003 ceph-mon[25122]: pgmap v478: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.1 MiB/s wr, 307 op/s 2024-06-01T02:32:19.681 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:19 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:32:20.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:19 smithi027 ceph-mon[27271]: pgmap v478: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.1 MiB/s wr, 307 op/s 2024-06-01T02:32:20.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:19 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:32:20.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 170767 7.95 MB/sec execute 361 sec latency 48.464 ms 2024-06-01T02:32:21.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 170985 7.96 MB/sec execute 362 sec latency 74.828 ms 2024-06-01T02:32:21.773 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:21 smithi003 ceph-mon[25122]: pgmap v479: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.6 MiB/s wr, 274 op/s 2024-06-01T02:32:22.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:21 smithi027 ceph-mon[27271]: pgmap v479: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.6 MiB/s wr, 274 op/s 2024-06-01T02:32:22.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 171178 7.95 MB/sec execute 363 sec latency 127.851 ms 2024-06-01T02:32:23.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 171584 7.96 MB/sec execute 364 sec latency 61.765 ms 2024-06-01T02:32:23.818 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:23 smithi003 ceph-mon[25122]: pgmap v480: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.9 MiB/s wr, 421 op/s 2024-06-01T02:32:24.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:23 smithi027 ceph-mon[27271]: pgmap v480: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.9 MiB/s wr, 421 op/s 2024-06-01T02:32:24.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 171927 7.95 MB/sec execute 365 sec latency 183.465 ms 2024-06-01T02:32:25.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:24 smithi003 ceph-mon[25122]: pgmap v481: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.9 MiB/s wr, 293 op/s 2024-06-01T02:32:25.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:32:25.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:32:25.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 172219 7.96 MB/sec execute 366 sec latency 151.363 ms 2024-06-01T02:32:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:24 smithi027 ceph-mon[27271]: pgmap v481: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.9 MiB/s wr, 293 op/s 2024-06-01T02:32:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:32:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:32:26.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 172420 7.94 MB/sec execute 367 sec latency 63.055 ms 2024-06-01T02:32:27.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 172777 7.94 MB/sec execute 368 sec latency 74.618 ms 2024-06-01T02:32:27.388 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:27 smithi003 ceph-mon[25122]: pgmap v482: 97 pgs: 97 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 419 op/s 2024-06-01T02:32:27.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:27 smithi027 ceph-mon[27271]: pgmap v482: 97 pgs: 97 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 419 op/s 2024-06-01T02:32:28.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 173028 7.93 MB/sec execute 369 sec latency 100.908 ms 2024-06-01T02:32:29.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 173250 7.94 MB/sec execute 370 sec latency 85.578 ms 2024-06-01T02:32:29.432 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:29 smithi003 ceph-mon[25122]: pgmap v483: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.6 MiB/s wr, 343 op/s 2024-06-01T02:32:29.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:29 smithi027 ceph-mon[27271]: pgmap v483: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.6 MiB/s wr, 343 op/s 2024-06-01T02:32:30.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 173614 7.95 MB/sec execute 371 sec latency 51.875 ms 2024-06-01T02:32:31.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 173929 7.95 MB/sec execute 372 sec latency 87.248 ms 2024-06-01T02:32:31.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:31 smithi003 ceph-mon[25122]: pgmap v484: 97 pgs: 97 active+clean; 352 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.2 MiB/s wr, 333 op/s 2024-06-01T02:32:31.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:31 smithi027 ceph-mon[27271]: pgmap v484: 97 pgs: 97 active+clean; 352 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.2 MiB/s wr, 333 op/s 2024-06-01T02:32:32.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 174225 7.95 MB/sec execute 373 sec latency 72.871 ms 2024-06-01T02:32:33.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 174442 7.95 MB/sec execute 374 sec latency 75.373 ms 2024-06-01T02:32:33.512 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:33 smithi003 ceph-mon[25122]: pgmap v485: 97 pgs: 97 active+clean; 352 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-06-01T02:32:33.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:33 smithi027 ceph-mon[27271]: pgmap v485: 97 pgs: 97 active+clean; 352 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-06-01T02:32:34.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 174751 7.95 MB/sec execute 375 sec latency 142.608 ms 2024-06-01T02:32:35.043 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:34 smithi003 ceph-mon[25122]: pgmap v486: 97 pgs: 97 active+clean; 352 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 312 op/s 2024-06-01T02:32:35.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 175052 7.95 MB/sec execute 376 sec latency 62.785 ms 2024-06-01T02:32:35.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:34 smithi027 ceph-mon[27271]: pgmap v486: 97 pgs: 97 active+clean; 352 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 312 op/s 2024-06-01T02:32:36.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 175380 7.95 MB/sec execute 377 sec latency 43.468 ms 2024-06-01T02:32:37.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 175607 7.94 MB/sec execute 378 sec latency 56.101 ms 2024-06-01T02:32:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:37 smithi003 ceph-mon[25122]: pgmap v487: 97 pgs: 97 active+clean; 356 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.8 MiB/s wr, 491 op/s 2024-06-01T02:32:37.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:37 smithi027 ceph-mon[27271]: pgmap v487: 97 pgs: 97 active+clean; 356 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.8 MiB/s wr, 491 op/s 2024-06-01T02:32:38.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 175878 7.94 MB/sec execute 379 sec latency 94.040 ms 2024-06-01T02:32:39.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 176170 7.93 MB/sec execute 380 sec latency 59.105 ms 2024-06-01T02:32:39.382 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:39 smithi003 ceph-mon[25122]: pgmap v488: 97 pgs: 97 active+clean; 358 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 385 op/s 2024-06-01T02:32:39.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:39 smithi027 ceph-mon[27271]: pgmap v488: 97 pgs: 97 active+clean; 358 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 385 op/s 2024-06-01T02:32:40.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 176542 7.93 MB/sec execute 381 sec latency 60.621 ms 2024-06-01T02:32:41.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 176843 7.95 MB/sec execute 382 sec latency 106.829 ms 2024-06-01T02:32:41.435 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:41 smithi003 ceph-mon[25122]: pgmap v489: 97 pgs: 97 active+clean; 354 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.6 MiB/s wr, 358 op/s 2024-06-01T02:32:41.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:41 smithi027 ceph-mon[27271]: pgmap v489: 97 pgs: 97 active+clean; 354 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.6 MiB/s wr, 358 op/s 2024-06-01T02:32:42.283 INFO:teuthology.orchestra.run.smithi003.stdout: 5 177262 7.96 MB/sec execute 383 sec latency 48.082 ms 2024-06-01T02:32:43.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 177458 7.95 MB/sec execute 384 sec latency 199.656 ms 2024-06-01T02:32:43.360 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:43 smithi003 ceph-mon[25122]: pgmap v490: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.3 MiB/s wr, 518 op/s 2024-06-01T02:32:43.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:43 smithi027 ceph-mon[27271]: pgmap v490: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.3 MiB/s wr, 518 op/s 2024-06-01T02:32:44.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 177792 7.95 MB/sec execute 385 sec latency 64.323 ms 2024-06-01T02:32:45.156 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:44 smithi003 ceph-mon[25122]: pgmap v491: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.1 MiB/s wr, 406 op/s 2024-06-01T02:32:45.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 178043 7.95 MB/sec execute 386 sec latency 90.991 ms 2024-06-01T02:32:45.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:44 smithi027 ceph-mon[27271]: pgmap v491: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.1 MiB/s wr, 406 op/s 2024-06-01T02:32:46.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 178332 7.95 MB/sec execute 387 sec latency 160.654 ms 2024-06-01T02:32:47.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 178617 7.95 MB/sec execute 388 sec latency 156.134 ms 2024-06-01T02:32:47.444 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:47 smithi003 ceph-mon[25122]: pgmap v492: 97 pgs: 97 active+clean; 354 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 507 op/s 2024-06-01T02:32:47.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:47 smithi027 ceph-mon[27271]: pgmap v492: 97 pgs: 97 active+clean; 354 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 507 op/s 2024-06-01T02:32:48.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 178891 7.96 MB/sec execute 389 sec latency 47.998 ms 2024-06-01T02:32:49.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 179250 7.96 MB/sec execute 390 sec latency 66.852 ms 2024-06-01T02:32:49.483 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:49 smithi003 ceph-mon[25122]: pgmap v493: 97 pgs: 97 active+clean; 356 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 350 op/s 2024-06-01T02:32:49.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:49 smithi027 ceph-mon[27271]: pgmap v493: 97 pgs: 97 active+clean; 356 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 350 op/s 2024-06-01T02:32:50.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 179592 7.94 MB/sec execute 391 sec latency 61.788 ms 2024-06-01T02:32:51.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 179885 7.94 MB/sec execute 392 sec latency 102.087 ms 2024-06-01T02:32:51.315 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:51 smithi003 ceph-mon[25122]: pgmap v494: 97 pgs: 97 active+clean; 362 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.3 MiB/s wr, 448 op/s 2024-06-01T02:32:51.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:51 smithi027 ceph-mon[27271]: pgmap v494: 97 pgs: 97 active+clean; 362 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.3 MiB/s wr, 448 op/s 2024-06-01T02:32:52.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 180118 7.94 MB/sec execute 393 sec latency 99.076 ms 2024-06-01T02:32:53.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 180358 7.94 MB/sec execute 394 sec latency 163.667 ms 2024-06-01T02:32:53.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:53 smithi003 ceph-mon[25122]: pgmap v495: 97 pgs: 97 active+clean; 354 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.1 MiB/s wr, 500 op/s 2024-06-01T02:32:53.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:53 smithi027 ceph-mon[27271]: pgmap v495: 97 pgs: 97 active+clean; 354 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.1 MiB/s wr, 500 op/s 2024-06-01T02:32:54.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 180607 7.94 MB/sec execute 395 sec latency 191.003 ms 2024-06-01T02:32:55.153 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:54 smithi003 ceph-mon[25122]: pgmap v496: 97 pgs: 97 active+clean; 352 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.5 MiB/s wr, 327 op/s 2024-06-01T02:32:55.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 180891 7.94 MB/sec execute 396 sec latency 100.277 ms 2024-06-01T02:32:55.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:54 smithi027 ceph-mon[27271]: pgmap v496: 97 pgs: 97 active+clean; 352 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.5 MiB/s wr, 327 op/s 2024-06-01T02:32:56.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 181107 7.94 MB/sec execute 397 sec latency 115.131 ms 2024-06-01T02:32:57.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 181301 7.93 MB/sec execute 398 sec latency 158.544 ms 2024-06-01T02:32:57.531 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:57 smithi003 ceph-mon[25122]: pgmap v497: 97 pgs: 97 active+clean; 353 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.5 MiB/s wr, 467 op/s 2024-06-01T02:32:57.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:57 smithi027 ceph-mon[27271]: pgmap v497: 97 pgs: 97 active+clean; 353 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.5 MiB/s wr, 467 op/s 2024-06-01T02:32:58.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 181476 7.93 MB/sec execute 399 sec latency 70.128 ms 2024-06-01T02:32:59.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 181683 7.92 MB/sec execute 400 sec latency 73.525 ms 2024-06-01T02:32:59.314 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:32:59 smithi003 ceph-mon[25122]: pgmap v498: 97 pgs: 97 active+clean; 355 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-06-01T02:32:59.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:32:59 smithi027 ceph-mon[27271]: pgmap v498: 97 pgs: 97 active+clean; 355 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-06-01T02:33:00.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 181959 7.92 MB/sec execute 401 sec latency 59.049 ms 2024-06-01T02:33:01.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 182208 7.92 MB/sec execute 402 sec latency 65.271 ms 2024-06-01T02:33:01.366 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:01 smithi003 ceph-mon[25122]: pgmap v499: 97 pgs: 97 active+clean; 355 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.4 MiB/s wr, 359 op/s 2024-06-01T02:33:01.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:01 smithi027 ceph-mon[27271]: pgmap v499: 97 pgs: 97 active+clean; 355 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.4 MiB/s wr, 359 op/s 2024-06-01T02:33:02.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 182462 7.92 MB/sec execute 403 sec latency 57.492 ms 2024-06-01T02:33:03.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 182729 7.92 MB/sec execute 404 sec latency 73.963 ms 2024-06-01T02:33:03.391 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:03 smithi003 ceph-mon[25122]: pgmap v500: 97 pgs: 97 active+clean; 355 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.2 MiB/s wr, 385 op/s 2024-06-01T02:33:03.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:03 smithi027 ceph-mon[27271]: pgmap v500: 97 pgs: 97 active+clean; 355 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.2 MiB/s wr, 385 op/s 2024-06-01T02:33:04.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 183000 7.90 MB/sec execute 405 sec latency 57.880 ms 2024-06-01T02:33:05.171 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:04 smithi003 ceph-mon[25122]: pgmap v501: 97 pgs: 97 active+clean; 354 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.5 MiB/s wr, 336 op/s 2024-06-01T02:33:05.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 183316 7.89 MB/sec execute 406 sec latency 80.066 ms 2024-06-01T02:33:05.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:05 smithi027 ceph-mon[27271]: pgmap v501: 97 pgs: 97 active+clean; 354 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.5 MiB/s wr, 336 op/s 2024-06-01T02:33:06.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 183571 7.89 MB/sec execute 407 sec latency 55.171 ms 2024-06-01T02:33:07.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 183913 7.90 MB/sec execute 408 sec latency 57.523 ms 2024-06-01T02:33:07.494 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:07 smithi003 ceph-mon[25122]: pgmap v502: 97 pgs: 97 active+clean; 356 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 469 op/s 2024-06-01T02:33:07.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:07 smithi027 ceph-mon[27271]: pgmap v502: 97 pgs: 97 active+clean; 356 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 469 op/s 2024-06-01T02:33:08.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 184337 7.92 MB/sec execute 409 sec latency 46.073 ms 2024-06-01T02:33:09.284 INFO:teuthology.orchestra.run.smithi003.stdout: 5 184722 7.92 MB/sec execute 410 sec latency 107.549 ms 2024-06-01T02:33:09.539 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:09 smithi003 ceph-mon[25122]: pgmap v503: 97 pgs: 97 active+clean; 357 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.6 MiB/s wr, 346 op/s 2024-06-01T02:33:09.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:09 smithi027 ceph-mon[27271]: pgmap v503: 97 pgs: 97 active+clean; 357 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.6 MiB/s wr, 346 op/s 2024-06-01T02:33:10.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 184959 7.92 MB/sec execute 411 sec latency 151.280 ms 2024-06-01T02:33:11.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 185259 7.92 MB/sec execute 412 sec latency 80.118 ms 2024-06-01T02:33:11.334 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:11 smithi003 ceph-mon[25122]: pgmap v504: 97 pgs: 97 active+clean; 357 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.1 MiB/s wr, 336 op/s 2024-06-01T02:33:11.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:11 smithi027 ceph-mon[27271]: pgmap v504: 97 pgs: 97 active+clean; 357 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.1 MiB/s wr, 336 op/s 2024-06-01T02:33:12.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 185552 7.92 MB/sec execute 413 sec latency 105.739 ms 2024-06-01T02:33:13.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 185774 7.92 MB/sec execute 414 sec latency 165.861 ms 2024-06-01T02:33:13.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:13 smithi003 ceph-mon[25122]: pgmap v505: 97 pgs: 97 active+clean; 360 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-06-01T02:33:13.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:13 smithi027 ceph-mon[27271]: pgmap v505: 97 pgs: 97 active+clean; 360 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-06-01T02:33:14.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 186013 7.92 MB/sec execute 415 sec latency 205.533 ms 2024-06-01T02:33:15.248 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:14 smithi003 ceph-mon[25122]: pgmap v506: 97 pgs: 97 active+clean; 360 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 335 op/s 2024-06-01T02:33:15.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 186200 7.91 MB/sec execute 416 sec latency 241.403 ms 2024-06-01T02:33:15.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:14 smithi027 ceph-mon[27271]: pgmap v506: 97 pgs: 97 active+clean; 360 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 335 op/s 2024-06-01T02:33:16.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 186531 7.91 MB/sec execute 417 sec latency 65.262 ms 2024-06-01T02:33:17.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 186749 7.89 MB/sec execute 418 sec latency 76.602 ms 2024-06-01T02:33:17.366 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:17 smithi003 ceph-mon[25122]: pgmap v507: 97 pgs: 97 active+clean; 361 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 479 op/s 2024-06-01T02:33:17.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:17 smithi027 ceph-mon[27271]: pgmap v507: 97 pgs: 97 active+clean; 361 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.7 MiB/s wr, 479 op/s 2024-06-01T02:33:18.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 186960 7.89 MB/sec execute 419 sec latency 91.819 ms 2024-06-01T02:33:19.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 187250 7.89 MB/sec execute 420 sec latency 86.495 ms 2024-06-01T02:33:19.438 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:19 smithi003 ceph-mon[25122]: pgmap v508: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 361 op/s 2024-06-01T02:33:19.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:19 smithi027 ceph-mon[27271]: pgmap v508: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 361 op/s 2024-06-01T02:33:20.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 187478 7.89 MB/sec execute 421 sec latency 62.851 ms 2024-06-01T02:33:21.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 187801 7.90 MB/sec execute 422 sec latency 58.987 ms 2024-06-01T02:33:21.532 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:21 smithi003 ceph-mon[25122]: pgmap v509: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.6 MiB/s wr, 335 op/s 2024-06-01T02:33:21.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:21 smithi027 ceph-mon[27271]: pgmap v509: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.6 MiB/s wr, 335 op/s 2024-06-01T02:33:22.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 188128 7.90 MB/sec execute 423 sec latency 99.936 ms 2024-06-01T02:33:23.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 188411 7.90 MB/sec execute 424 sec latency 55.554 ms 2024-06-01T02:33:23.323 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:23 smithi003 ceph-mon[25122]: pgmap v510: 97 pgs: 97 active+clean; 358 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.0 MiB/s wr, 487 op/s 2024-06-01T02:33:23.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:23 smithi027 ceph-mon[27271]: pgmap v510: 97 pgs: 97 active+clean; 358 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.0 MiB/s wr, 487 op/s 2024-06-01T02:33:24.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 188688 7.90 MB/sec execute 425 sec latency 99.124 ms 2024-06-01T02:33:25.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:24 smithi003 ceph-mon[25122]: pgmap v511: 97 pgs: 97 active+clean; 358 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.4 MiB/s wr, 354 op/s 2024-06-01T02:33:25.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:33:25.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:24 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:33:25.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 188959 7.89 MB/sec execute 426 sec latency 86.832 ms 2024-06-01T02:33:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:24 smithi027 ceph-mon[27271]: pgmap v511: 97 pgs: 97 active+clean; 358 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.4 MiB/s wr, 354 op/s 2024-06-01T02:33:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:33:25.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:24 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:33:26.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 189152 7.89 MB/sec execute 427 sec latency 61.951 ms 2024-06-01T02:33:27.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 189359 7.89 MB/sec execute 428 sec latency 72.866 ms 2024-06-01T02:33:27.517 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:27 smithi003 ceph-mon[25122]: pgmap v512: 97 pgs: 97 active+clean; 354 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-06-01T02:33:27.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:27 smithi027 ceph-mon[27271]: pgmap v512: 97 pgs: 97 active+clean; 354 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-06-01T02:33:28.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 189565 7.89 MB/sec execute 429 sec latency 98.000 ms 2024-06-01T02:33:29.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 189765 7.87 MB/sec execute 430 sec latency 75.128 ms 2024-06-01T02:33:29.374 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:29 smithi003 ceph-mon[25122]: pgmap v513: 97 pgs: 97 active+clean; 356 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.8 MiB/s wr, 378 op/s 2024-06-01T02:33:29.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:29 smithi027 ceph-mon[27271]: pgmap v513: 97 pgs: 97 active+clean; 356 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.8 MiB/s wr, 378 op/s 2024-06-01T02:33:30.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 190021 7.87 MB/sec execute 431 sec latency 101.839 ms 2024-06-01T02:33:31.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 190299 7.86 MB/sec execute 432 sec latency 64.634 ms 2024-06-01T02:33:31.437 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:31 smithi003 ceph-mon[25122]: pgmap v514: 97 pgs: 97 active+clean; 356 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.5 MiB/s wr, 357 op/s 2024-06-01T02:33:31.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:31 smithi027 ceph-mon[27271]: pgmap v514: 97 pgs: 97 active+clean; 356 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.5 MiB/s wr, 357 op/s 2024-06-01T02:33:32.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 190614 7.86 MB/sec execute 433 sec latency 78.326 ms 2024-06-01T02:33:33.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 190868 7.86 MB/sec execute 434 sec latency 101.633 ms 2024-06-01T02:33:33.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:33 smithi003 ceph-mon[25122]: pgmap v515: 97 pgs: 97 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 453 op/s 2024-06-01T02:33:33.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:33 smithi027 ceph-mon[27271]: pgmap v515: 97 pgs: 97 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 453 op/s 2024-06-01T02:33:34.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191179 7.87 MB/sec execute 435 sec latency 125.893 ms 2024-06-01T02:33:35.243 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:34 smithi003 ceph-mon[25122]: pgmap v516: 97 pgs: 97 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.2 MiB/s wr, 294 op/s 2024-06-01T02:33:35.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191611 7.88 MB/sec execute 436 sec latency 110.852 ms 2024-06-01T02:33:35.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:35 smithi027 ceph-mon[27271]: pgmap v516: 97 pgs: 97 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.2 MiB/s wr, 294 op/s 2024-06-01T02:33:36.285 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191732 7.87 MB/sec execute 437 sec latency 122.501 ms 2024-06-01T02:33:37.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191792 7.86 MB/sec execute 438 sec latency 348.791 ms 2024-06-01T02:33:37.584 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:37 smithi003 ceph-mon[25122]: pgmap v517: 97 pgs: 97 active+clean; 357 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 427 op/s 2024-06-01T02:33:37.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:37 smithi027 ceph-mon[27271]: pgmap v517: 97 pgs: 97 active+clean; 357 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 427 op/s 2024-06-01T02:33:38.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191856 7.84 MB/sec execute 439 sec latency 257.042 ms 2024-06-01T02:33:39.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191885 7.82 MB/sec execute 440 sec latency 413.636 ms 2024-06-01T02:33:39.478 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:39 smithi003 ceph-mon[25122]: pgmap v518: 97 pgs: 97 active+clean; 357 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 392 op/s 2024-06-01T02:33:39.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:39 smithi027 ceph-mon[27271]: pgmap v518: 97 pgs: 97 active+clean; 357 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 392 op/s 2024-06-01T02:33:40.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191908 7.81 MB/sec execute 441 sec latency 371.550 ms 2024-06-01T02:33:41.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191915 7.79 MB/sec execute 442 sec latency 621.888 ms 2024-06-01T02:33:41.584 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:41 smithi003 ceph-mon[25122]: pgmap v519: 97 pgs: 97 active+clean; 359 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.1 MiB/s wr, 317 op/s 2024-06-01T02:33:41.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:41 smithi027 ceph-mon[27271]: pgmap v519: 97 pgs: 97 active+clean; 359 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.1 MiB/s wr, 317 op/s 2024-06-01T02:33:42.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191919 7.77 MB/sec execute 443 sec latency 1425.719 ms 2024-06-01T02:33:43.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191929 7.75 MB/sec execute 444 sec latency 1849.667 ms 2024-06-01T02:33:43.584 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:43 smithi003 ceph-mon[25122]: pgmap v520: 97 pgs: 97 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.2 MiB/s wr, 326 op/s 2024-06-01T02:33:43.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:43 smithi027 ceph-mon[27271]: pgmap v520: 97 pgs: 97 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.2 MiB/s wr, 326 op/s 2024-06-01T02:33:44.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191929 7.74 MB/sec execute 445 sec latency 1576.640 ms 2024-06-01T02:33:45.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191929 7.72 MB/sec execute 446 sec latency 2576.685 ms 2024-06-01T02:33:45.334 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:45 smithi003 ceph-mon[25122]: pgmap v521: 97 pgs: 97 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.3 MiB/s wr, 229 op/s 2024-06-01T02:33:45.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:45 smithi027 ceph-mon[27271]: pgmap v521: 97 pgs: 97 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.3 MiB/s wr, 229 op/s 2024-06-01T02:33:46.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.70 MB/sec execute 447 sec latency 2915.842 ms 2024-06-01T02:33:47.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.69 MB/sec execute 448 sec latency 1660.897 ms 2024-06-01T02:33:47.584 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:47 smithi003 ceph-mon[25122]: pgmap v522: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.2 MiB/s wr, 227 op/s 2024-06-01T02:33:47.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:47 smithi027 ceph-mon[27271]: pgmap v522: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.2 MiB/s wr, 227 op/s 2024-06-01T02:33:48.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.67 MB/sec execute 449 sec latency 2660.942 ms 2024-06-01T02:33:49.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.65 MB/sec execute 450 sec latency 3661.001 ms 2024-06-01T02:33:49.584 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:49 smithi003 ceph-mon[25122]: pgmap v523: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 949 KiB/s wr, 86 op/s 2024-06-01T02:33:49.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:49 smithi027 ceph-mon[27271]: pgmap v523: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.5 KiB/s rd, 949 KiB/s wr, 86 op/s 2024-06-01T02:33:50.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.64 MB/sec execute 451 sec latency 4661.048 ms 2024-06-01T02:33:51.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.62 MB/sec execute 452 sec latency 5661.087 ms 2024-06-01T02:33:51.584 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:51 smithi003 ceph-mon[25122]: pgmap v524: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 KiB/s rd, 465 KiB/s wr, 64 op/s 2024-06-01T02:33:51.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:51 smithi027 ceph-mon[27271]: pgmap v524: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 KiB/s rd, 465 KiB/s wr, 64 op/s 2024-06-01T02:33:52.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.60 MB/sec execute 453 sec latency 6661.132 ms 2024-06-01T02:33:53.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.58 MB/sec execute 454 sec latency 7661.177 ms 2024-06-01T02:33:53.584 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:53 smithi003 ceph-mon[25122]: pgmap v525: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 853 B/s rd, 243 KiB/s wr, 18 op/s 2024-06-01T02:33:53.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:53 smithi027 ceph-mon[27271]: pgmap v525: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 853 B/s rd, 243 KiB/s wr, 18 op/s 2024-06-01T02:33:54.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.57 MB/sec execute 455 sec latency 8661.232 ms 2024-06-01T02:33:55.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.55 MB/sec execute 456 sec latency 9661.258 ms 2024-06-01T02:33:55.334 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:54 smithi003 ceph-mon[25122]: pgmap v526: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 57 KiB/s wr, 7 op/s 2024-06-01T02:33:55.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:54 smithi027 ceph-mon[27271]: pgmap v526: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 57 KiB/s wr, 7 op/s 2024-06-01T02:33:56.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.53 MB/sec execute 457 sec latency 10661.306 ms 2024-06-01T02:33:57.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.52 MB/sec execute 458 sec latency 11661.357 ms 2024-06-01T02:33:57.584 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:57 smithi003 ceph-mon[25122]: pgmap v527: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 56 KiB/s wr, 6 op/s 2024-06-01T02:33:57.600 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:57 smithi027 ceph-mon[27271]: pgmap v527: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 56 KiB/s wr, 6 op/s 2024-06-01T02:33:58.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.50 MB/sec execute 459 sec latency 12661.407 ms 2024-06-01T02:33:59.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.49 MB/sec execute 460 sec latency 13661.461 ms 2024-06-01T02:34:00.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:59 smithi003 ceph-mon[25122]: pgmap v528: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 17 KiB/s wr, 1 op/s 2024-06-01T02:34:00.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:59 smithi003 ceph-mon[25122]: pgmap v529: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 197 B/s rd, 888 B/s wr, 0 op/s 2024-06-01T02:34:00.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:59 smithi003 ceph-mon[25122]: pgmap v530: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 244 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-06-01T02:34:00.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:00.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:00.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:33:59 smithi003 ceph-mon[25122]: Health check cleared: CEPHADM_HOST_CHECK_FAILED (was: 1 hosts fail cephadm check) 2024-06-01T02:34:00.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:59 smithi027 ceph-mon[27271]: pgmap v528: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 17 KiB/s wr, 1 op/s 2024-06-01T02:34:00.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:59 smithi027 ceph-mon[27271]: pgmap v529: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 197 B/s rd, 888 B/s wr, 0 op/s 2024-06-01T02:34:00.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:59 smithi027 ceph-mon[27271]: pgmap v530: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 244 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-06-01T02:34:00.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:00.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:00.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:33:59 smithi027 ceph-mon[27271]: Health check cleared: CEPHADM_HOST_CHECK_FAILED (was: 1 hosts fail cephadm check) 2024-06-01T02:34:00.286 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.47 MB/sec execute 461 sec latency 14661.487 ms 2024-06-01T02:34:01.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.45 MB/sec execute 462 sec latency 15661.546 ms 2024-06-01T02:34:02.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:01 smithi003 ceph-mon[25122]: pgmap v531: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 B/s wr, 0 op/s 2024-06-01T02:34:02.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:01 smithi027 ceph-mon[27271]: pgmap v531: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 B/s wr, 0 op/s 2024-06-01T02:34:02.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.44 MB/sec execute 463 sec latency 16661.606 ms 2024-06-01T02:34:03.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.42 MB/sec execute 464 sec latency 17661.646 ms 2024-06-01T02:34:03.834 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:03 smithi003 ceph-mon[25122]: pgmap v532: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 244 B/s rd, 0 B/s wr, 0 op/s 2024-06-01T02:34:04.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:03 smithi027 ceph-mon[27271]: pgmap v532: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 244 B/s rd, 0 B/s wr, 0 op/s 2024-06-01T02:34:04.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.41 MB/sec execute 465 sec latency 18661.679 ms 2024-06-01T02:34:05.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.39 MB/sec execute 466 sec latency 19661.722 ms 2024-06-01T02:34:05.334 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:05 smithi003 ceph-mon[25122]: pgmap v533: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 244 B/s rd, 0 op/s 2024-06-01T02:34:05.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:05 smithi027 ceph-mon[27271]: pgmap v533: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 244 B/s rd, 0 op/s 2024-06-01T02:34:06.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.37 MB/sec execute 467 sec latency 20661.765 ms 2024-06-01T02:34:06.835 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:34:06 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:34:06.577+0000 7fc8a3a71700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:40.838294+0000 front 2024-06-01T02:33:40.838155+0000 (oldest deadline 2024-06-01T02:34:05.756453+0000) 2024-06-01T02:34:07.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.36 MB/sec execute 468 sec latency 21661.814 ms 2024-06-01T02:34:07.600 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:34:07 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:34:07.160+0000 7f5697eaf700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:41.462468+0000 front 2024-06-01T02:33:41.462421+0000 (oldest deadline 2024-06-01T02:34:06.745064+0000) 2024-06-01T02:34:07.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:07 smithi003 ceph-mon[25122]: pgmap v534: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 op/s 2024-06-01T02:34:07.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:07 smithi003 ceph-mon[25122]: osd.2 reported failed by osd.1 2024-06-01T02:34:07.835 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:34:07 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:34:07.558+0000 7fc8a3a71700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:40.838294+0000 front 2024-06-01T02:33:40.838155+0000 (oldest deadline 2024-06-01T02:34:05.756453+0000) 2024-06-01T02:34:07.835 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:34:07 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:34:07.461+0000 7f0a76d24700 -1 osd.1 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623533+0000 front 2024-06-01T02:33:45.623089+0000 (oldest deadline 2024-06-01T02:34:07.008703+0000) 2024-06-01T02:34:08.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:07 smithi027 ceph-mon[27271]: pgmap v534: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 122 B/s rd, 0 op/s 2024-06-01T02:34:08.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:07 smithi027 ceph-mon[27271]: osd.2 reported failed by osd.1 2024-06-01T02:34:08.101 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:34:07 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:34:07.673+0000 7fe29e3d7700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623059+0000 front 2024-06-01T02:33:45.623210+0000 (oldest deadline 2024-06-01T02:34:07.018495+0000) 2024-06-01T02:34:08.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.34 MB/sec execute 469 sec latency 22661.842 ms 2024-06-01T02:34:08.600 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:34:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:34:08.208+0000 7f5697eaf700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:41.462468+0000 front 2024-06-01T02:33:41.462421+0000 (oldest deadline 2024-06-01T02:34:06.745064+0000) 2024-06-01T02:34:08.829 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:34:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:34:08.482+0000 7f0a76d24700 -1 osd.1 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623533+0000 front 2024-06-01T02:33:45.623089+0000 (oldest deadline 2024-06-01T02:34:07.008703+0000) 2024-06-01T02:34:08.830 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:34:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:34:08.509+0000 7fc8a3a71700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:40.838294+0000 front 2024-06-01T02:33:40.838155+0000 (oldest deadline 2024-06-01T02:34:05.756453+0000) 2024-06-01T02:34:09.085 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:34:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:34:08.831+0000 7fdb99b79700 -1 osd.0 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:42.711888+0000 front 2024-06-01T02:33:42.710683+0000 (oldest deadline 2024-06-01T02:34:08.605823+0000) 2024-06-01T02:34:09.100 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:34:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:34:08.717+0000 7fe29e3d7700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623059+0000 front 2024-06-01T02:33:45.623210+0000 (oldest deadline 2024-06-01T02:34:07.018495+0000) 2024-06-01T02:34:09.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.33 MB/sec execute 470 sec latency 23661.869 ms 2024-06-01T02:34:09.601 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:34:09 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:34:09.213+0000 7f5697eaf700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:41.462468+0000 front 2024-06-01T02:33:41.462421+0000 (oldest deadline 2024-06-01T02:34:06.745064+0000) 2024-06-01T02:34:09.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:09 smithi003 ceph-mon[25122]: pgmap v535: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:34:09.835 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:34:09 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:34:09.496+0000 7f0a76d24700 -1 osd.1 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623533+0000 front 2024-06-01T02:33:45.623089+0000 (oldest deadline 2024-06-01T02:34:07.008703+0000) 2024-06-01T02:34:09.835 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:34:09 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:34:09.479+0000 7fc8a3a71700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:40.838294+0000 front 2024-06-01T02:33:40.838155+0000 (oldest deadline 2024-06-01T02:34:05.756453+0000) 2024-06-01T02:34:10.054 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:09 smithi027 ceph-mon[27271]: pgmap v535: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:34:10.054 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:34:09 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:34:09.695+0000 7fe29e3d7700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623059+0000 front 2024-06-01T02:33:45.623210+0000 (oldest deadline 2024-06-01T02:34:07.018495+0000) 2024-06-01T02:34:10.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.31 MB/sec execute 471 sec latency 24661.897 ms 2024-06-01T02:34:10.335 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:34:09 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:34:09.853+0000 7fdb99b79700 -1 osd.0 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:42.711888+0000 front 2024-06-01T02:33:42.710683+0000 (oldest deadline 2024-06-01T02:34:08.605823+0000) 2024-06-01T02:34:10.350 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:34:10.201+0000 7f5697eaf700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:41.462468+0000 front 2024-06-01T02:33:41.462421+0000 (oldest deadline 2024-06-01T02:34:06.745064+0000) 2024-06-01T02:34:10.351 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[40459]: 2024-06-01T02:34:10.054+0000 7fd552b8d700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623609+0000 front 2024-06-01T02:33:45.623037+0000 (oldest deadline 2024-06-01T02:34:09.724113+0000) 2024-06-01T02:34:10.835 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:34:10.473+0000 7f0a76d24700 -1 osd.1 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623533+0000 front 2024-06-01T02:33:45.623089+0000 (oldest deadline 2024-06-01T02:34:07.008703+0000) 2024-06-01T02:34:10.835 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:34:10.465+0000 7fc8a3a71700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:40.838294+0000 front 2024-06-01T02:33:40.838155+0000 (oldest deadline 2024-06-01T02:34:05.756453+0000) 2024-06-01T02:34:10.851 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:34:10.743+0000 7fe29e3d7700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623059+0000 front 2024-06-01T02:33:45.623210+0000 (oldest deadline 2024-06-01T02:34:07.018495+0000) 2024-06-01T02:34:10.851 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[37402]: 2024-06-01T02:34:10.531+0000 7fb321db6700 -1 osd.6 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:45.623253+0000 front 2024-06-01T02:33:45.623629+0000 (oldest deadline 2024-06-01T02:34:10.213096+0000) 2024-06-01T02:34:11.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191939 7.30 MB/sec execute 472 sec latency 25661.928 ms 2024-06-01T02:34:11.335 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:34:10.861+0000 7fdb99b79700 -1 osd.0 70 heartbeat_check: no reply from 172.21.15.3:6822 osd.2 since back 2024-06-01T02:33:42.711888+0000 front 2024-06-01T02:33:42.710683+0000 (oldest deadline 2024-06-01T02:34:08.605823+0000) 2024-06-01T02:34:11.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:11.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-mon[25122]: osd.2 reported failed by osd.4 2024-06-01T02:34:11.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-mon[25122]: osd.2 failed (root=default,host=smithi003) (2 reporters from different osd after 23.740598 >= grace 20.000000) 2024-06-01T02:34:11.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-mon[25122]: osd.2 reported failed by osd.3 2024-06-01T02:34:11.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-mon[25122]: pgmap v536: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:34:11.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-mon[25122]: osd.2 reported failed by osd.5 2024-06-01T02:34:11.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:10 smithi003 ceph-mon[25122]: osd.2 reported failed by osd.0 2024-06-01T02:34:11.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:11.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-mon[27271]: osd.2 reported failed by osd.4 2024-06-01T02:34:11.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-mon[27271]: osd.2 failed (root=default,host=smithi003) (2 reporters from different osd after 23.740598 >= grace 20.000000) 2024-06-01T02:34:11.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-mon[27271]: osd.2 reported failed by osd.3 2024-06-01T02:34:11.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-mon[27271]: pgmap v536: 97 pgs: 1 active+clean+laggy, 96 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:34:11.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-mon[27271]: osd.2 reported failed by osd.5 2024-06-01T02:34:11.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:10 smithi027 ceph-mon[27271]: osd.2 reported failed by osd.0 2024-06-01T02:34:12.213 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:11 smithi003 ceph-mon[25122]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:34:12.214 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:11 smithi003 ceph-mon[25122]: osdmap e71: 8 total, 7 up, 8 in 2024-06-01T02:34:12.214 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:11 smithi003 ceph-mon[25122]: osdmap e72: 8 total, 7 up, 8 in 2024-06-01T02:34:12.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 191964 7.28 MB/sec execute 473 sec latency 26308.479 ms 2024-06-01T02:34:12.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:11 smithi027 ceph-mon[27271]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:34:12.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:11 smithi027 ceph-mon[27271]: osdmap e71: 8 total, 7 up, 8 in 2024-06-01T02:34:12.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:11 smithi027 ceph-mon[27271]: osdmap e72: 8 total, 7 up, 8 in 2024-06-01T02:34:12.997 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:12 smithi003 ceph-mon[25122]: pgmap v539: 97 pgs: 11 active+undersized+degraded, 7 peering, 4 active+undersized, 75 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 383 B/s rd, 112 KiB/s wr, 3 op/s; 75/1593 objects degraded (4.708%) 2024-06-01T02:34:13.252 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:12 smithi027 ceph-mon[27271]: pgmap v539: 97 pgs: 11 active+undersized+degraded, 7 peering, 4 active+undersized, 75 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 383 B/s rd, 112 KiB/s wr, 3 op/s; 75/1593 objects degraded (4.708%) 2024-06-01T02:34:13.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 192105 7.28 MB/sec execute 474 sec latency 103.912 ms 2024-06-01T02:34:14.050 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:13 smithi003 ceph-mon[25122]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-06-01T02:34:14.051 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:13 smithi003 ceph-mon[25122]: Health check failed: Degraded data redundancy: 75/1593 objects degraded (4.708%), 11 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:14.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 192328 7.27 MB/sec execute 475 sec latency 141.980 ms 2024-06-01T02:34:14.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:13 smithi027 ceph-mon[27271]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-06-01T02:34:14.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:13 smithi027 ceph-mon[27271]: Health check failed: Degraded data redundancy: 75/1593 objects degraded (4.708%), 11 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:15.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:14 smithi003 ceph-mon[25122]: pgmap v540: 97 pgs: 11 active+undersized+degraded, 7 peering, 4 active+undersized, 75 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 383 B/s rd, 112 KiB/s wr, 3 op/s; 75/1593 objects degraded (4.708%) 2024-06-01T02:34:15.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 192641 7.27 MB/sec execute 476 sec latency 129.130 ms 2024-06-01T02:34:15.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:14 smithi027 ceph-mon[27271]: pgmap v540: 97 pgs: 11 active+undersized+degraded, 7 peering, 4 active+undersized, 75 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 383 B/s rd, 112 KiB/s wr, 3 op/s; 75/1593 objects degraded (4.708%) 2024-06-01T02:34:16.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 192901 7.27 MB/sec execute 477 sec latency 152.418 ms 2024-06-01T02:34:16.874 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:16 smithi003 ceph-mon[25122]: pgmap v541: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 358 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 1.2 MiB/s wr, 124 op/s; 145/1668 objects degraded (8.693%) 2024-06-01T02:34:16.875 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:16 smithi003 ceph-mon[25122]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-06-01T02:34:17.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:16 smithi027 ceph-mon[27271]: pgmap v541: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 358 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 1.2 MiB/s wr, 124 op/s; 145/1668 objects degraded (8.693%) 2024-06-01T02:34:17.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:16 smithi027 ceph-mon[27271]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-06-01T02:34:17.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 193199 7.28 MB/sec execute 478 sec latency 62.403 ms 2024-06-01T02:34:18.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 193598 7.28 MB/sec execute 479 sec latency 54.133 ms 2024-06-01T02:34:18.942 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:18 smithi003 ceph-mon[25122]: pgmap v542: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 353 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 155 op/s; 147/1677 objects degraded (8.766%) 2024-06-01T02:34:18.943 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:18 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 147/1677 objects degraded (8.766%), 19 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:19.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:18 smithi027 ceph-mon[27271]: pgmap v542: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 353 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.9 MiB/s wr, 155 op/s; 147/1677 objects degraded (8.766%) 2024-06-01T02:34:19.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:18 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 147/1677 objects degraded (8.766%), 19 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:19.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 193927 7.27 MB/sec execute 480 sec latency 61.096 ms 2024-06-01T02:34:20.287 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194166 7.27 MB/sec execute 481 sec latency 120.535 ms 2024-06-01T02:34:21.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194446 7.28 MB/sec execute 482 sec latency 116.456 ms 2024-06-01T02:34:22.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:21 smithi003 ceph-mon[25122]: pgmap v543: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 355 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.0 MiB/s wr, 315 op/s; 150/1650 objects degraded (9.091%) 2024-06-01T02:34:22.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:21 smithi027 ceph-mon[27271]: pgmap v543: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 355 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.0 MiB/s wr, 315 op/s; 150/1650 objects degraded (9.091%) 2024-06-01T02:34:22.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194451 7.26 MB/sec execute 483 sec latency 1106.141 ms 2024-06-01T02:34:23.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:22 smithi003 ceph-mon[25122]: pgmap v544: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.2 MiB/s wr, 322 op/s; 145/1602 objects degraded (9.051%) 2024-06-01T02:34:23.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:22 smithi027 ceph-mon[27271]: pgmap v544: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.2 MiB/s wr, 322 op/s; 145/1602 objects degraded (9.051%) 2024-06-01T02:34:23.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194451 7.25 MB/sec execute 484 sec latency 2106.178 ms 2024-06-01T02:34:24.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.23 MB/sec execute 485 sec latency 3101.520 ms 2024-06-01T02:34:24.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:24 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 145/1602 objects degraded (9.051%), 19 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:24.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:24 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 145/1602 objects degraded (9.051%), 19 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:25.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.22 MB/sec execute 486 sec latency 4101.570 ms 2024-06-01T02:34:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:25 smithi003 ceph-mon[25122]: pgmap v545: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.8 MiB/s wr, 285 op/s; 145/1602 objects degraded (9.051%) 2024-06-01T02:34:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:34:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:34:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:25 smithi027 ceph-mon[27271]: pgmap v545: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 355 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.8 MiB/s wr, 285 op/s; 145/1602 objects degraded (9.051%) 2024-06-01T02:34:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:34:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:34:26.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.20 MB/sec execute 487 sec latency 5101.628 ms 2024-06-01T02:34:27.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.19 MB/sec execute 488 sec latency 6101.673 ms 2024-06-01T02:34:28.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:27 smithi003 ceph-mon[25122]: pgmap v546: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 354 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.0 MiB/s wr, 312 op/s; 142/1551 objects degraded (9.155%) 2024-06-01T02:34:28.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:27 smithi027 ceph-mon[27271]: pgmap v546: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 354 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.0 MiB/s wr, 312 op/s; 142/1551 objects degraded (9.155%) 2024-06-01T02:34:28.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.17 MB/sec execute 489 sec latency 7101.716 ms 2024-06-01T02:34:29.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.16 MB/sec execute 490 sec latency 8101.770 ms 2024-06-01T02:34:30.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:29 smithi003 ceph-mon[25122]: pgmap v547: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.2 MiB/s wr, 232 op/s; 141/1542 objects degraded (9.144%) 2024-06-01T02:34:30.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:29 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 141/1542 objects degraded (9.144%), 19 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:30.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:29 smithi027 ceph-mon[27271]: pgmap v547: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 350 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.2 MiB/s wr, 232 op/s; 141/1542 objects degraded (9.144%) 2024-06-01T02:34:30.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:29 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 141/1542 objects degraded (9.144%), 19 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:30.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.15 MB/sec execute 491 sec latency 9101.819 ms 2024-06-01T02:34:31.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.13 MB/sec execute 492 sec latency 10101.848 ms 2024-06-01T02:34:32.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:31 smithi003 ceph-mon[25122]: pgmap v548: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.8 MiB/s wr, 211 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:32.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:31 smithi027 ceph-mon[27271]: pgmap v548: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.8 MiB/s wr, 211 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:32.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.12 MB/sec execute 493 sec latency 11101.877 ms 2024-06-01T02:34:33.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.10 MB/sec execute 494 sec latency 12101.926 ms 2024-06-01T02:34:33.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:33 smithi003 ceph-mon[25122]: pgmap v549: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 670 KiB/s wr, 60 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:34.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:33 smithi027 ceph-mon[27271]: pgmap v549: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 670 KiB/s wr, 60 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:34.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.09 MB/sec execute 495 sec latency 13101.965 ms 2024-06-01T02:34:35.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:34 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 141/1539 objects degraded (9.162%), 19 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:35.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:34 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 141/1539 objects degraded (9.162%), 19 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:35.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.07 MB/sec execute 496 sec latency 14101.991 ms 2024-06-01T02:34:36.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:35 smithi003 ceph-mon[25122]: pgmap v550: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 197 KiB/s wr, 27 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:36.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:35 smithi027 ceph-mon[27271]: pgmap v550: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 197 KiB/s wr, 27 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:36.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.06 MB/sec execute 497 sec latency 15102.022 ms 2024-06-01T02:34:37.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.04 MB/sec execute 498 sec latency 16102.046 ms 2024-06-01T02:34:38.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:37 smithi003 ceph-mon[25122]: pgmap v551: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 197 KiB/s wr, 27 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:38.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:37 smithi027 ceph-mon[27271]: pgmap v551: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 197 KiB/s wr, 27 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:38.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.03 MB/sec execute 499 sec latency 17102.076 ms 2024-06-01T02:34:39.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.02 MB/sec execute 500 sec latency 18102.123 ms 2024-06-01T02:34:40.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:39 smithi003 ceph-mon[25122]: pgmap v552: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 426 B/s wr, 0 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:40.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:39 smithi027 ceph-mon[27271]: pgmap v552: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 426 B/s wr, 0 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:40.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 7.00 MB/sec execute 501 sec latency 19102.165 ms 2024-06-01T02:34:41.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 6.99 MB/sec execute 502 sec latency 20102.205 ms 2024-06-01T02:34:42.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:41 smithi003 ceph-mon[25122]: pgmap v553: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:42.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:41 smithi027 ceph-mon[27271]: pgmap v553: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:42.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 6.97 MB/sec execute 503 sec latency 21102.249 ms 2024-06-01T02:34:43.002 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:42 smithi027 ceph-mon[27271]: pgmap v554: 97 pgs: 1 active+clean+laggy, 19 active+undersized+degraded, 7 active+undersized, 70 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:43.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:42 smithi003 ceph-mon[25122]: pgmap v554: 97 pgs: 1 active+clean+laggy, 19 active+undersized+degraded, 7 active+undersized, 70 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:43.288 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 6.96 MB/sec execute 504 sec latency 22102.296 ms 2024-06-01T02:34:44.071 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:34:43 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:34:43.679+0000 7fe29e3d7700 -1 osd.5 72 heartbeat_check: no reply from 172.21.15.3:6814 osd.1 since back 2024-06-01T02:34:19.823225+0000 front 2024-06-01T02:34:19.822978+0000 (oldest deadline 2024-06-01T02:34:43.323131+0000) 2024-06-01T02:34:44.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 6.95 MB/sec execute 505 sec latency 23102.339 ms 2024-06-01T02:34:44.351 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:34:44 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[40459]: 2024-06-01T02:34:44.072+0000 7fd552b8d700 -1 osd.7 72 heartbeat_check: no reply from 172.21.15.3:6814 osd.1 since back 2024-06-01T02:34:19.831228+0000 front 2024-06-01T02:34:19.831147+0000 (oldest deadline 2024-06-01T02:34:43.331236+0000) 2024-06-01T02:34:45.054 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:34:44 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:34:44.653+0000 7fe29e3d7700 -1 osd.5 72 heartbeat_check: no reply from 172.21.15.3:6814 osd.1 since back 2024-06-01T02:34:19.823225+0000 front 2024-06-01T02:34:19.822978+0000 (oldest deadline 2024-06-01T02:34:43.323131+0000) 2024-06-01T02:34:45.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194452 6.93 MB/sec execute 506 sec latency 24102.381 ms 2024-06-01T02:34:45.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:45 smithi003 ceph-mon[25122]: osd.1 reported failed by osd.7 2024-06-01T02:34:45.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:45 smithi003 ceph-mon[25122]: pgmap v555: 97 pgs: 1 active+clean+laggy, 19 active+undersized+degraded, 7 active+undersized, 70 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 B/s rd, 0 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:45.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:45 smithi003 ceph-mon[25122]: osd.1 reported failed by osd.5 2024-06-01T02:34:45.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:45 smithi003 ceph-mon[25122]: osd.1 failed (root=default,host=smithi003) (2 reporters from different osd after 24.000168 >= grace 20.000000) 2024-06-01T02:34:45.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:45 smithi027 ceph-mon[27271]: osd.1 reported failed by osd.7 2024-06-01T02:34:45.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:45 smithi027 ceph-mon[27271]: pgmap v555: 97 pgs: 1 active+clean+laggy, 19 active+undersized+degraded, 7 active+undersized, 70 active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 B/s rd, 0 op/s; 141/1539 objects degraded (9.162%) 2024-06-01T02:34:45.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:45 smithi027 ceph-mon[27271]: osd.1 reported failed by osd.5 2024-06-01T02:34:45.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:45 smithi027 ceph-mon[27271]: osd.1 failed (root=default,host=smithi003) (2 reporters from different osd after 24.000168 >= grace 20.000000) 2024-06-01T02:34:46.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.92 MB/sec execute 507 sec latency 24860.436 ms 2024-06-01T02:34:46.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:46 smithi003 ceph-mon[25122]: Health check update: 2 osds down (OSD_DOWN) 2024-06-01T02:34:46.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:46 smithi003 ceph-mon[25122]: osdmap e73: 8 total, 6 up, 8 in 2024-06-01T02:34:46.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:46 smithi003 ceph-mon[25122]: osdmap e74: 8 total, 6 up, 8 in 2024-06-01T02:34:46.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:46 smithi027 ceph-mon[27271]: Health check update: 2 osds down (OSD_DOWN) 2024-06-01T02:34:46.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:46 smithi027 ceph-mon[27271]: osdmap e73: 8 total, 6 up, 8 in 2024-06-01T02:34:46.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:46 smithi027 ceph-mon[27271]: osdmap e74: 8 total, 6 up, 8 in 2024-06-01T02:34:47.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.91 MB/sec execute 508 sec latency 1216.013 ms 2024-06-01T02:34:47.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:47 smithi003 ceph-mon[25122]: pgmap v558: 97 pgs: 9 peering, 1 stale+active+undersized, 52 active+clean, 1 undersized+peered, 5 active+undersized, 1 undersized+degraded+peered, 20 active+undersized+degraded, 1 active+clean+laggy, 6 stale+active+clean, 1 stale+active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 78 KiB/s wr, 8 op/s; 167/1518 objects degraded (11.001%) 2024-06-01T02:34:47.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:47 smithi027 ceph-mon[27271]: pgmap v558: 97 pgs: 9 peering, 1 stale+active+undersized, 52 active+clean, 1 undersized+peered, 5 active+undersized, 1 undersized+degraded+peered, 20 active+undersized+degraded, 1 active+clean+laggy, 6 stale+active+clean, 1 stale+active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 78 KiB/s wr, 8 op/s; 167/1518 objects degraded (11.001%) 2024-06-01T02:34:48.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.90 MB/sec execute 509 sec latency 2216.039 ms 2024-06-01T02:34:48.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:48 smithi003 ceph-mon[25122]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-06-01T02:34:48.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:48 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 167/1518 objects degraded (11.001%), 22 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:48.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:48 smithi027 ceph-mon[27271]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-06-01T02:34:48.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:48 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 167/1518 objects degraded (11.001%), 22 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:49.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.88 MB/sec execute 510 sec latency 3216.065 ms 2024-06-01T02:34:49.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:49 smithi003 ceph-mon[25122]: pgmap v559: 97 pgs: 3 activating+undersized, 9 peering, 1 stale+active+undersized, 44 active+clean, 2 undersized+peered, 8 active+undersized, 3 undersized+degraded+peered, 23 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 78 KiB/s wr, 9 op/s; 214/1506 objects degraded (14.210%) 2024-06-01T02:34:49.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:49 smithi027 ceph-mon[27271]: pgmap v559: 97 pgs: 3 activating+undersized, 9 peering, 1 stale+active+undersized, 44 active+clean, 2 undersized+peered, 8 active+undersized, 3 undersized+degraded+peered, 23 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 78 KiB/s wr, 9 op/s; 214/1506 objects degraded (14.210%) 2024-06-01T02:34:50.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.87 MB/sec execute 511 sec latency 4216.110 ms 2024-06-01T02:34:51.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.86 MB/sec execute 512 sec latency 5216.138 ms 2024-06-01T02:34:52.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:51 smithi003 ceph-mon[25122]: pgmap v560: 97 pgs: 4 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 28 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 281 KiB/s wr, 13 op/s; 272/1500 objects degraded (18.133%) 2024-06-01T02:34:52.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:51 smithi027 ceph-mon[27271]: pgmap v560: 97 pgs: 4 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 28 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 281 KiB/s wr, 13 op/s; 272/1500 objects degraded (18.133%) 2024-06-01T02:34:52.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.84 MB/sec execute 513 sec latency 6216.188 ms 2024-06-01T02:34:53.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:52 smithi003 ceph-mon[25122]: pgmap v561: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 296 KiB/s wr, 18 op/s; 272/1500 objects degraded (18.133%) 2024-06-01T02:34:53.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:52 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 272/1500 objects degraded (18.133%), 33 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:53.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:52 smithi027 ceph-mon[27271]: pgmap v561: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 296 KiB/s wr, 18 op/s; 272/1500 objects degraded (18.133%) 2024-06-01T02:34:53.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:52 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 272/1500 objects degraded (18.133%), 33 pgs degraded (PG_DEGRADED) 2024-06-01T02:34:53.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.83 MB/sec execute 514 sec latency 7216.228 ms 2024-06-01T02:34:54.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.82 MB/sec execute 515 sec latency 8216.268 ms 2024-06-01T02:34:54.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:54 smithi003 ceph-mon[25122]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-06-01T02:34:54.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:54 smithi027 ceph-mon[27271]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-06-01T02:34:55.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.80 MB/sec execute 516 sec latency 9216.316 ms 2024-06-01T02:34:55.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:55 smithi003 ceph-mon[25122]: pgmap v562: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 247 KiB/s wr, 15 op/s; 272/1500 objects degraded (18.133%) 2024-06-01T02:34:55.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:55 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:55.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:55 smithi027 ceph-mon[27271]: pgmap v562: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 247 KiB/s wr, 15 op/s; 272/1500 objects degraded (18.133%) 2024-06-01T02:34:55.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:55 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:34:56.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.79 MB/sec execute 517 sec latency 10216.356 ms 2024-06-01T02:34:57.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.78 MB/sec execute 518 sec latency 11216.380 ms 2024-06-01T02:34:58.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:57 smithi003 ceph-mon[25122]: pgmap v563: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 234 KiB/s wr, 15 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:34:58.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:57 smithi027 ceph-mon[27271]: pgmap v563: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 234 KiB/s wr, 15 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:34:58.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.76 MB/sec execute 519 sec latency 12216.408 ms 2024-06-01T02:34:58.968 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:34:58.968 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:34:58.968 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:34:59.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:34:59.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:34:59.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:34:59.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.75 MB/sec execute 520 sec latency 13216.432 ms 2024-06-01T02:35:00.043 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:59 smithi027 ceph-mon[27271]: pgmap v564: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 155 KiB/s wr, 8 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:00.043 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:34:59 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 272/1497 objects degraded (18.170%), 33 pgs degraded (PG_DEGRADED) 2024-06-01T02:35:00.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:59 smithi003 ceph-mon[25122]: pgmap v564: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 155 KiB/s wr, 8 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:00.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:34:59 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 272/1497 objects degraded (18.170%), 33 pgs degraded (PG_DEGRADED) 2024-06-01T02:35:00.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.74 MB/sec execute 521 sec latency 14216.477 ms 2024-06-01T02:35:01.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.72 MB/sec execute 522 sec latency 15216.502 ms 2024-06-01T02:35:01.887 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:01 smithi027 ceph-mon[27271]: pgmap v565: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 155 KiB/s wr, 7 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:01.888 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:01 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:35:02.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:01 smithi003 ceph-mon[25122]: pgmap v565: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 155 KiB/s wr, 7 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:02.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:01 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:35:02.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.71 MB/sec execute 523 sec latency 16216.554 ms 2024-06-01T02:35:03.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:35:03.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:35:03.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:35:03.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:35:03.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:02 smithi003 ceph-mon[25122]: Adjusting osd_memory_target on smithi027 to 2704M 2024-06-01T02:35:03.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:35:03.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:02 smithi003 ceph-mon[25122]: pgmap v566: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s wr, 4 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:03.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:35:03.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:35:03.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:35:03.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-01T02:35:03.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:02 smithi027 ceph-mon[27271]: Adjusting osd_memory_target on smithi027 to 2704M 2024-06-01T02:35:03.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:35:03.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:02 smithi027 ceph-mon[27271]: pgmap v566: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s wr, 4 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:03.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.70 MB/sec execute 524 sec latency 17216.580 ms 2024-06-01T02:35:04.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.69 MB/sec execute 525 sec latency 18216.606 ms 2024-06-01T02:35:05.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.67 MB/sec execute 526 sec latency 19216.636 ms 2024-06-01T02:35:05.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:05 smithi003 ceph-mon[25122]: pgmap v567: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s wr, 1 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:05.350 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:05 smithi027 ceph-mon[27271]: pgmap v567: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s wr, 1 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:06.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.66 MB/sec execute 527 sec latency 20216.685 ms 2024-06-01T02:35:07.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.65 MB/sec execute 528 sec latency 21216.732 ms 2024-06-01T02:35:07.956 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:07 smithi027 ceph-mon[27271]: pgmap v568: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s wr, 1 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:08.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:07 smithi003 ceph-mon[25122]: pgmap v568: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s wr, 1 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:08.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.64 MB/sec execute 529 sec latency 22216.783 ms 2024-06-01T02:35:08.351 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:35:07 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:35:07.957+0000 7f5697eaf700 -1 osd.4 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:47.303074+0000 front 2024-06-01T02:34:46.057165+0000 (oldest deadline 2024-06-01T02:35:07.055441+0000) 2024-06-01T02:35:08.968 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:35:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:35:08.709+0000 7fdb99b79700 -1 osd.0 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:51.047189+0000 front 2024-06-01T02:34:42.911323+0000 (oldest deadline 2024-06-01T02:35:08.211387+0000) 2024-06-01T02:35:09.101 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:35:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:35:08.915+0000 7f5697eaf700 -1 osd.4 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:47.303074+0000 front 2024-06-01T02:34:46.057165+0000 (oldest deadline 2024-06-01T02:35:07.055441+0000) 2024-06-01T02:35:09.101 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:35:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:35:08.672+0000 7fe29e3d7700 -1 osd.5 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:41.826917+0000 front 2024-06-01T02:34:41.842009+0000 (oldest deadline 2024-06-01T02:35:07.727044+0000) 2024-06-01T02:35:09.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.62 MB/sec execute 530 sec latency 23216.827 ms 2024-06-01T02:35:09.921 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:09 smithi027 ceph-mon[27271]: pgmap v569: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:09.921 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:35:09 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:35:09.626+0000 7fe29e3d7700 -1 osd.5 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:41.826917+0000 front 2024-06-01T02:34:41.842009+0000 (oldest deadline 2024-06-01T02:35:07.727044+0000) 2024-06-01T02:35:10.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:09 smithi003 ceph-mon[25122]: pgmap v569: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:10.085 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:35:09 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:35:09.668+0000 7fdb99b79700 -1 osd.0 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:51.047189+0000 front 2024-06-01T02:34:42.911323+0000 (oldest deadline 2024-06-01T02:35:08.211387+0000) 2024-06-01T02:35:10.289 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.61 MB/sec execute 531 sec latency 24216.878 ms 2024-06-01T02:35:10.351 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:35:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[37402]: 2024-06-01T02:35:10.161+0000 7fb321db6700 -1 osd.6 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:43.523086+0000 front 2024-06-01T02:34:51.049677+0000 (oldest deadline 2024-06-01T02:35:09.423217+0000) 2024-06-01T02:35:10.351 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:35:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[40459]: 2024-06-01T02:35:10.103+0000 7fd552b8d700 -1 osd.7 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:43.734893+0000 front 2024-06-01T02:34:43.734994+0000 (oldest deadline 2024-06-01T02:35:09.634945+0000) 2024-06-01T02:35:10.351 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:35:09 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:35:09.923+0000 7f5697eaf700 -1 osd.4 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:47.303074+0000 front 2024-06-01T02:34:46.057165+0000 (oldest deadline 2024-06-01T02:35:07.055441+0000) 2024-06-01T02:35:10.851 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:35:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:35:10.581+0000 7fe29e3d7700 -1 osd.5 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:41.826917+0000 front 2024-06-01T02:34:41.842009+0000 (oldest deadline 2024-06-01T02:35:07.727044+0000) 2024-06-01T02:35:11.085 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:35:10 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:35:10.637+0000 7fdb99b79700 -1 osd.0 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:51.047189+0000 front 2024-06-01T02:34:42.911323+0000 (oldest deadline 2024-06-01T02:35:08.211387+0000) 2024-06-01T02:35:11.196 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:35:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[40459]: 2024-06-01T02:35:11.084+0000 7fd552b8d700 -1 osd.7 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:43.734893+0000 front 2024-06-01T02:34:43.734994+0000 (oldest deadline 2024-06-01T02:35:09.634945+0000) 2024-06-01T02:35:11.196 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:35:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:35:10.943+0000 7f5697eaf700 -1 osd.4 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:47.303074+0000 front 2024-06-01T02:34:46.057165+0000 (oldest deadline 2024-06-01T02:35:07.055441+0000) 2024-06-01T02:35:11.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.60 MB/sec execute 532 sec latency 25216.905 ms 2024-06-01T02:35:11.601 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:35:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[37402]: 2024-06-01T02:35:11.197+0000 7fb321db6700 -1 osd.6 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:43.523086+0000 front 2024-06-01T02:34:51.049677+0000 (oldest deadline 2024-06-01T02:35:09.423217+0000) 2024-06-01T02:35:12.085 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:35:11 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:35:11.593+0000 7fdb99b79700 -1 osd.0 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:51.047189+0000 front 2024-06-01T02:34:42.911323+0000 (oldest deadline 2024-06-01T02:35:08.211387+0000) 2024-06-01T02:35:12.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:11 smithi003 ceph-mon[25122]: pgmap v570: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:12.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:11 smithi003 ceph-mon[25122]: osd.3 reported failed by osd.7 2024-06-01T02:35:12.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:11 smithi003 ceph-mon[25122]: osd.3 reported failed by osd.6 2024-06-01T02:35:12.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:11 smithi003 ceph-mon[25122]: osd.3 failed (root=default,host=smithi003) (2 reporters from different osd after 27.000156 >= grace 20.000000) 2024-06-01T02:35:12.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:11 smithi003 ceph-mon[25122]: osd.3 reported failed by osd.5 2024-06-01T02:35:12.101 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:35:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:35:11.621+0000 7fe29e3d7700 -1 osd.5 74 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:34:41.826917+0000 front 2024-06-01T02:34:41.842009+0000 (oldest deadline 2024-06-01T02:35:07.727044+0000) 2024-06-01T02:35:12.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:11 smithi027 ceph-mon[27271]: pgmap v570: 97 pgs: 3 activating+undersized, 1 stale+active+undersized, 44 active+clean, 3 undersized+peered, 9 active+undersized, 4 undersized+degraded+peered, 29 active+undersized+degraded, 1 active+clean+laggy, 3 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s wr, 0 op/s; 272/1497 objects degraded (18.170%) 2024-06-01T02:35:12.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:11 smithi027 ceph-mon[27271]: osd.3 reported failed by osd.7 2024-06-01T02:35:12.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:11 smithi027 ceph-mon[27271]: osd.3 reported failed by osd.6 2024-06-01T02:35:12.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:11 smithi027 ceph-mon[27271]: osd.3 failed (root=default,host=smithi003) (2 reporters from different osd after 27.000156 >= grace 20.000000) 2024-06-01T02:35:12.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:11 smithi027 ceph-mon[27271]: osd.3 reported failed by osd.5 2024-06-01T02:35:12.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194486 6.59 MB/sec execute 533 sec latency 26216.945 ms 2024-06-01T02:35:12.967 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:12 smithi003 ceph-mon[25122]: Health check update: 3 osds down (OSD_DOWN) 2024-06-01T02:35:12.967 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:12 smithi003 ceph-mon[25122]: osdmap e75: 8 total, 5 up, 8 in 2024-06-01T02:35:12.968 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:12 smithi003 ceph-mon[25122]: osdmap e76: 8 total, 5 up, 8 in 2024-06-01T02:35:13.003 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:12 smithi027 ceph-mon[27271]: Health check update: 3 osds down (OSD_DOWN) 2024-06-01T02:35:13.003 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:12 smithi027 ceph-mon[27271]: osdmap e75: 8 total, 5 up, 8 in 2024-06-01T02:35:13.003 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:12 smithi027 ceph-mon[27271]: osdmap e76: 8 total, 5 up, 8 in 2024-06-01T02:35:13.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194502 6.58 MB/sec execute 534 sec latency 27216.971 ms 2024-06-01T02:35:14.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:13 smithi003 ceph-mon[25122]: pgmap v572: 97 pgs: 14 peering, 4 stale+active+undersized+degraded, 1 stale+active+clean+laggy, 1 activating+undersized, 2 stale+active+undersized, 28 active+clean, 3 undersized+peered, 7 active+undersized, 4 undersized+degraded+peered, 22 active+undersized+degraded, 11 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s wr, 0 op/s; 254/1488 objects degraded (17.070%) 2024-06-01T02:35:14.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:13 smithi003 ceph-mon[25122]: Health check update: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-06-01T02:35:14.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:13 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 254/1488 objects degraded (17.070%), 30 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-06-01T02:35:14.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:13 smithi027 ceph-mon[27271]: pgmap v572: 97 pgs: 14 peering, 4 stale+active+undersized+degraded, 1 stale+active+clean+laggy, 1 activating+undersized, 2 stale+active+undersized, 28 active+clean, 3 undersized+peered, 7 active+undersized, 4 undersized+degraded+peered, 22 active+undersized+degraded, 11 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s wr, 0 op/s; 254/1488 objects degraded (17.070%) 2024-06-01T02:35:14.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:13 smithi027 ceph-mon[27271]: Health check update: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-06-01T02:35:14.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:13 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 254/1488 objects degraded (17.070%), 30 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-06-01T02:35:14.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.56 MB/sec execute 535 sec latency 28216.998 ms 2024-06-01T02:35:15.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.55 MB/sec execute 536 sec latency 29217.034 ms 2024-06-01T02:35:15.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:15 smithi003 ceph-mon[25122]: pgmap v574: 97 pgs: 14 peering, 4 stale+active+undersized+degraded, 1 stale+active+clean+laggy, 1 activating+undersized, 2 stale+active+undersized, 28 active+clean, 3 undersized+peered, 7 active+undersized, 4 undersized+degraded+peered, 22 active+undersized+degraded, 11 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 254/1488 objects degraded (17.070%) 2024-06-01T02:35:15.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:15 smithi027 ceph-mon[27271]: pgmap v574: 97 pgs: 14 peering, 4 stale+active+undersized+degraded, 1 stale+active+clean+laggy, 1 activating+undersized, 2 stale+active+undersized, 28 active+clean, 3 undersized+peered, 7 active+undersized, 4 undersized+degraded+peered, 22 active+undersized+degraded, 11 stale+active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 254/1488 objects degraded (17.070%) 2024-06-01T02:35:16.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.54 MB/sec execute 537 sec latency 30217.061 ms 2024-06-01T02:35:17.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.53 MB/sec execute 538 sec latency 31217.107 ms 2024-06-01T02:35:18.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:17 smithi003 ceph-mon[25122]: pgmap v575: 97 pgs: 14 peering, 1 stale+active+undersized, 20 active+clean, 5 undersized+peered, 10 active+undersized, 13 undersized+degraded+peered, 34 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s wr, 10 op/s; 422/1473 objects degraded (28.649%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:18.100 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:17 smithi027 ceph-mon[27271]: pgmap v575: 97 pgs: 14 peering, 1 stale+active+undersized, 20 active+clean, 5 undersized+peered, 10 active+undersized, 13 undersized+degraded+peered, 34 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s wr, 10 op/s; 422/1473 objects degraded (28.649%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:18.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.51 MB/sec execute 539 sec latency 32217.154 ms 2024-06-01T02:35:18.967 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:18 smithi003 ceph-mon[25122]: pgmap v576: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 138 KiB/s wr, 12 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:18.968 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:18 smithi003 ceph-mon[25122]: Health check update: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2024-06-01T02:35:18.968 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:18 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 500/1467 objects degraded (34.083%), 53 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-06-01T02:35:19.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:18 smithi027 ceph-mon[27271]: pgmap v576: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 138 KiB/s wr, 12 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:19.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:18 smithi027 ceph-mon[27271]: Health check update: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2024-06-01T02:35:19.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:18 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 500/1467 objects degraded (34.083%), 53 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-06-01T02:35:19.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.50 MB/sec execute 540 sec latency 33217.180 ms 2024-06-01T02:35:20.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.49 MB/sec execute 541 sec latency 34217.229 ms 2024-06-01T02:35:21.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.48 MB/sec execute 542 sec latency 35217.271 ms 2024-06-01T02:35:22.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:21 smithi003 ceph-mon[25122]: pgmap v577: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 123 KiB/s wr, 10 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:22.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:21 smithi027 ceph-mon[27271]: pgmap v577: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 123 KiB/s wr, 10 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:22.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.47 MB/sec execute 543 sec latency 36217.322 ms 2024-06-01T02:35:23.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:22 smithi003 ceph-mon[25122]: pgmap v578: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 110 KiB/s wr, 9 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:23.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:22 smithi003 ceph-mon[25122]: Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO) 2024-06-01T02:35:23.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:22 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:23.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:22 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:23.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:22 smithi027 ceph-mon[27271]: pgmap v578: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 110 KiB/s wr, 9 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:23.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:22 smithi027 ceph-mon[27271]: Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO) 2024-06-01T02:35:23.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:22 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:23.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:22 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:23.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.45 MB/sec execute 544 sec latency 37217.365 ms 2024-06-01T02:35:24.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.44 MB/sec execute 545 sec latency 38217.413 ms 2024-06-01T02:35:25.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.43 MB/sec execute 546 sec latency 39217.440 ms 2024-06-01T02:35:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:25 smithi003 ceph-mon[25122]: pgmap v579: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 92 KiB/s wr, 8 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:35:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:35:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:35:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:25 smithi027 ceph-mon[27271]: pgmap v579: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 92 KiB/s wr, 8 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:35:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:35:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:35:26.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.42 MB/sec execute 547 sec latency 40217.470 ms 2024-06-01T02:35:27.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.41 MB/sec execute 548 sec latency 41217.495 ms 2024-06-01T02:35:27.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:27 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:27.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:27 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:27.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:27 smithi003 ceph-mon[25122]: pgmap v580: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s wr, 8 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:27.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:27 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:27.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:27 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:27.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:27 smithi027 ceph-mon[27271]: pgmap v580: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 95 KiB/s wr, 8 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:28.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.40 MB/sec execute 549 sec latency 42217.539 ms 2024-06-01T02:35:29.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.38 MB/sec execute 550 sec latency 43217.582 ms 2024-06-01T02:35:30.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:29 smithi003 ceph-mon[25122]: pgmap v581: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s wr, 1 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:30.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:29 smithi027 ceph-mon[27271]: pgmap v581: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 27 KiB/s wr, 1 op/s; 500/1467 objects degraded (34.083%); 0 B/s, 0 objects/s recovering 2024-06-01T02:35:30.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.37 MB/sec execute 551 sec latency 44217.632 ms 2024-06-01T02:35:31.290 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.36 MB/sec execute 552 sec latency 45217.680 ms 2024-06-01T02:35:32.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:31 smithi003 ceph-mon[25122]: pgmap v582: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:32.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:31 smithi027 ceph-mon[27271]: pgmap v582: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:32.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.35 MB/sec execute 553 sec latency 46217.728 ms 2024-06-01T02:35:33.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.34 MB/sec execute 554 sec latency 47217.772 ms 2024-06-01T02:35:33.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:33 smithi003 ceph-mon[25122]: pgmap v583: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:34.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:33 smithi027 ceph-mon[27271]: pgmap v583: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:34.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.33 MB/sec execute 555 sec latency 48217.822 ms 2024-06-01T02:35:35.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.32 MB/sec execute 556 sec latency 49217.872 ms 2024-06-01T02:35:35.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:35 smithi003 ceph-mon[25122]: pgmap v584: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:35.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:35 smithi027 ceph-mon[27271]: pgmap v584: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:36.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.30 MB/sec execute 557 sec latency 50217.898 ms 2024-06-01T02:35:36.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:36 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:36.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:36 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:36.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:36 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:36.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:36 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:37.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.29 MB/sec execute 558 sec latency 51217.924 ms 2024-06-01T02:35:37.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:37 smithi003 ceph-mon[25122]: pgmap v585: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:37.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:37 smithi027 ceph-mon[27271]: pgmap v585: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:38.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.28 MB/sec execute 559 sec latency 52217.979 ms 2024-06-01T02:35:39.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.27 MB/sec execute 560 sec latency 53218.004 ms 2024-06-01T02:35:39.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:39 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:39.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:39 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:39.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:39 smithi003 ceph-mon[25122]: pgmap v586: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:39.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:39 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:39.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:39 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:39.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:39 smithi027 ceph-mon[27271]: pgmap v586: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:40.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.26 MB/sec execute 561 sec latency 54218.030 ms 2024-06-01T02:35:41.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.25 MB/sec execute 562 sec latency 55218.075 ms 2024-06-01T02:35:42.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:41 smithi003 ceph-mon[25122]: pgmap v587: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:42.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:41 smithi027 ceph-mon[27271]: pgmap v587: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:42.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.24 MB/sec execute 563 sec latency 56218.126 ms 2024-06-01T02:35:43.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.23 MB/sec execute 564 sec latency 57218.169 ms 2024-06-01T02:35:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:43 smithi003 ceph-mon[25122]: pgmap v588: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:43 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:43.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:43 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:44.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:43 smithi027 ceph-mon[27271]: pgmap v588: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:44.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:43 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:44.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:43 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:44.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.22 MB/sec execute 565 sec latency 58218.219 ms 2024-06-01T02:35:45.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.20 MB/sec execute 566 sec latency 59218.263 ms 2024-06-01T02:35:45.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:45 smithi003 ceph-mon[25122]: pgmap v589: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:45.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:45 smithi027 ceph-mon[27271]: pgmap v589: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:46.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.19 MB/sec execute 567 sec latency 60218.311 ms 2024-06-01T02:35:47.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.18 MB/sec execute 568 sec latency 61218.337 ms 2024-06-01T02:35:47.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:47 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:47.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:47 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:47.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:47 smithi003 ceph-mon[25122]: pgmap v590: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:47.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:47 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:47.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:47 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:47.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:47 smithi027 ceph-mon[27271]: pgmap v590: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:48.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.17 MB/sec execute 569 sec latency 62218.401 ms 2024-06-01T02:35:48.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:48 smithi003 ceph-mon[25122]: Health check update: Reduced data availability: 14 pgs inactive, 1 pg stale (PG_AVAILABILITY) 2024-06-01T02:35:48.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:48 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 500/1467 objects degraded (34.083%), 53 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-06-01T02:35:48.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:48 smithi027 ceph-mon[27271]: Health check update: Reduced data availability: 14 pgs inactive, 1 pg stale (PG_AVAILABILITY) 2024-06-01T02:35:48.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:48 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 500/1467 objects degraded (34.083%), 53 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2024-06-01T02:35:49.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.16 MB/sec execute 570 sec latency 63218.426 ms 2024-06-01T02:35:49.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:49 smithi003 ceph-mon[25122]: pgmap v591: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:49.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:49 smithi027 ceph-mon[27271]: pgmap v591: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:50.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.15 MB/sec execute 571 sec latency 64218.451 ms 2024-06-01T02:35:51.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.14 MB/sec execute 572 sec latency 65218.497 ms 2024-06-01T02:35:52.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:51 smithi003 ceph-mon[25122]: pgmap v592: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:52.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:51 smithi027 ceph-mon[27271]: pgmap v592: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:52.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.13 MB/sec execute 573 sec latency 66218.527 ms 2024-06-01T02:35:53.291 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.12 MB/sec execute 574 sec latency 67218.583 ms 2024-06-01T02:35:53.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:53 smithi003 ceph-mon[25122]: pgmap v593: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:54.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:53 smithi027 ceph-mon[27271]: pgmap v593: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:54.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.11 MB/sec execute 575 sec latency 68218.633 ms 2024-06-01T02:35:55.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.10 MB/sec execute 576 sec latency 69218.682 ms 2024-06-01T02:35:55.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:55 smithi003 ceph-mon[25122]: pgmap v594: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:55.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:55 smithi027 ceph-mon[27271]: pgmap v594: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:56.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.09 MB/sec execute 577 sec latency 70218.733 ms 2024-06-01T02:35:56.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:56 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:56.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:56 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:56.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:56 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:56.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:56 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:57.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.08 MB/sec execute 578 sec latency 71218.779 ms 2024-06-01T02:35:57.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:57 smithi003 ceph-mon[25122]: pgmap v595: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:57.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:57 smithi027 ceph-mon[27271]: pgmap v595: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:58.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.06 MB/sec execute 579 sec latency 72218.823 ms 2024-06-01T02:35:59.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.05 MB/sec execute 580 sec latency 73218.865 ms 2024-06-01T02:35:59.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:59 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:59.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:59 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:59.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:35:59 smithi003 ceph-mon[25122]: pgmap v596: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:35:59.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:59 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:35:59.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:59 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:35:59.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:35:59 smithi027 ceph-mon[27271]: pgmap v596: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:36:00.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.04 MB/sec execute 581 sec latency 74218.946 ms 2024-06-01T02:36:01.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.03 MB/sec execute 582 sec latency 75219.009 ms 2024-06-01T02:36:01.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:01 smithi003 ceph-mon[25122]: pgmap v597: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:36:02.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:01 smithi027 ceph-mon[27271]: pgmap v597: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:36:02.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.02 MB/sec execute 583 sec latency 76219.320 ms 2024-06-01T02:36:03.028 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:02 smithi003 ceph-mon[25122]: osd.2 marked itself dead as of e76 2024-06-01T02:36:03.028 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:36:02 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:36:02.198+0000 7f0a76d24700 -1 osd.1 72 heartbeat_check: no reply from 172.21.15.27:6804 osd.4 since back 2024-06-01T02:34:23.290091+0000 front 2024-06-01T02:34:16.320754+0000 (oldest deadline 2024-06-01T02:34:43.286129+0000) 2024-06-01T02:36:03.028 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:36:02 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:36:02.198+0000 7f0a76d24700 -1 osd.1 72 heartbeat_check: no reply from 172.21.15.27:6820 osd.6 since back 2024-06-01T02:34:16.320711+0000 front 2024-06-01T02:34:23.290206+0000 (oldest deadline 2024-06-01T02:34:43.286129+0000) 2024-06-01T02:36:03.029 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:36:02 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[42096]: 2024-06-01T02:36:01.998+0000 7ff4d65b8700 -1 osd.2 70 get_health_metrics reporting 3 slow ops, oldest is osd_repop(client.14562.0:111344 3.9 e70/62 3:951029a9:::10000008599.00000000:head v 70'2893, mlcod=70'2889) 2024-06-01T02:36:03.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:02 smithi027 ceph-mon[27271]: osd.2 marked itself dead as of e76 2024-06-01T02:36:03.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.01 MB/sec execute 584 sec latency 77219.374 ms 2024-06-01T02:36:03.335 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:36:02.199+0000 7f0a76d24700 -1 osd.1 72 get_health_metrics reporting 24 slow ops, oldest is osd_op(mds.0.4:213855 2.6 2:6572afc8:::200.000000ab:head [write 68485~35937 [fadvise_dontneed] in=35937b] snapc 0=[] ondisk+write+known_if_redirected+full_force+supports_pool_eio e72) 2024-06-01T02:36:03.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-mon[25122]: pgmap v598: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:36:03.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-mon[25122]: Health check failed: 3 slow ops, oldest one blocked for 136 sec, osd.2 has slow ops (SLOW_OPS) 2024-06-01T02:36:03.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-mon[25122]: osdmap e77: 8 total, 5 up, 8 in 2024-06-01T02:36:03.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:36:03.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:36:03.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-mon[25122]: osd.6 failure report canceled by osd.1 2024-06-01T02:36:03.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-mon[25122]: osd.4 failure report canceled by osd.1 2024-06-01T02:36:03.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-mon[25122]: osd.1 marked itself dead as of e77 2024-06-01T02:36:03.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:03 smithi003 ceph-mon[25122]: osd.3 marked itself dead as of e77 2024-06-01T02:36:04.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:03 smithi027 ceph-mon[27271]: pgmap v598: 97 pgs: 1 stale+active+undersized, 20 active+clean, 10 undersized+peered, 13 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:36:04.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:03 smithi027 ceph-mon[27271]: Health check failed: 3 slow ops, oldest one blocked for 136 sec, osd.2 has slow ops (SLOW_OPS) 2024-06-01T02:36:04.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:03 smithi027 ceph-mon[27271]: osdmap e77: 8 total, 5 up, 8 in 2024-06-01T02:36:04.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:03 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:36:04.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:03 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:36:04.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:03 smithi027 ceph-mon[27271]: osd.6 failure report canceled by osd.1 2024-06-01T02:36:04.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:03 smithi027 ceph-mon[27271]: osd.4 failure report canceled by osd.1 2024-06-01T02:36:04.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:03 smithi027 ceph-mon[27271]: osd.1 marked itself dead as of e77 2024-06-01T02:36:04.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:03 smithi027 ceph-mon[27271]: osd.3 marked itself dead as of e77 2024-06-01T02:36:04.294 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 6.00 MB/sec execute 585 sec latency 78219.466 ms 2024-06-01T02:36:04.725 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:36:04 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:36:04.568+0000 7f0a75521700 -1 osd.1 78 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:36:04.726 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:36:04 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[42096]: 2024-06-01T02:36:04.410+0000 7ff4d4db5700 -1 osd.2 77 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:36:05.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:04 smithi003 ceph-mon[25122]: osdmap e78: 8 total, 5 up, 8 in 2024-06-01T02:36:05.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:04 smithi003 ceph-mon[25122]: pgmap v601: 97 pgs: 20 active+clean, 10 undersized+peered, 14 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:36:05.085 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:36:04 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:36:04.818+0000 7fc8a226e700 -1 osd.3 78 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:36:05.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:04 smithi027 ceph-mon[27271]: osdmap e78: 8 total, 5 up, 8 in 2024-06-01T02:36:05.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:04 smithi027 ceph-mon[27271]: pgmap v601: 97 pgs: 20 active+clean, 10 undersized+peered, 14 active+undersized, 17 undersized+degraded+peered, 36 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 500/1467 objects degraded (34.083%) 2024-06-01T02:36:05.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194529 5.99 MB/sec execute 586 sec latency 79219.511 ms 2024-06-01T02:36:06.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: Health check cleared: OSD_DOWN (was: 3 osds down) 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: Health check failed: Slow OSD heartbeats on back (longest 37968.531ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: Health check failed: Slow OSD heartbeats on front (longest 37875.098ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: Health check update: Reduced data availability: 14 pgs inactive (PG_AVAILABILITY) 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357] boot 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156] boot 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546] boot 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: osdmap e79: 8 total, 8 up, 8 in 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:36:06.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:05 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:36:06.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: Health check cleared: OSD_DOWN (was: 3 osds down) 2024-06-01T02:36:06.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: Health check failed: Slow OSD heartbeats on back (longest 37968.531ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-01T02:36:06.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: Health check failed: Slow OSD heartbeats on front (longest 37875.098ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-01T02:36:06.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: Health check update: Reduced data availability: 14 pgs inactive (PG_AVAILABILITY) 2024-06-01T02:36:06.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: osd.2 [v2:172.21.15.3:6818/2180786357,v1:172.21.15.3:6819/2180786357] boot 2024-06-01T02:36:06.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: osd.1 [v2:172.21.15.3:6810/2291814156,v1:172.21.15.3:6811/2291814156] boot 2024-06-01T02:36:06.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: osd.3 [v2:172.21.15.3:6826/1979857546,v1:172.21.15.3:6827/1979857546] boot 2024-06-01T02:36:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: osdmap e79: 8 total, 8 up, 8 in 2024-06-01T02:36:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:36:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:36:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:05 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:36:06.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194533 5.98 MB/sec execute 587 sec latency 80184.171 ms 2024-06-01T02:36:07.266 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:07 smithi003 ceph-mon[25122]: Monitor daemon marked osd.2 down, but it is still running 2024-06-01T02:36:07.266 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:07 smithi003 ceph-mon[25122]: map e76 wrongly marked me down at e71 2024-06-01T02:36:07.266 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:07 smithi003 ceph-mon[25122]: Monitor daemon marked osd.1 down, but it is still running 2024-06-01T02:36:07.266 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:07 smithi003 ceph-mon[25122]: map e77 wrongly marked me down at e73 2024-06-01T02:36:07.266 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:07 smithi003 ceph-mon[25122]: osdmap e80: 8 total, 8 up, 8 in 2024-06-01T02:36:07.266 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:07 smithi003 ceph-mon[25122]: MDS health message cleared (mds.?): 7 slow metadata IOs are blocked > 30 secs, oldest blocked for 73 secs 2024-06-01T02:36:07.266 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:07 smithi003 ceph-mon[25122]: pgmap v604: 97 pgs: 1 activating+degraded, 16 peering, 26 active+clean, 9 undersized+peered, 10 active+undersized, 17 undersized+degraded+peered, 18 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 354/1467 objects degraded (24.131%) 2024-06-01T02:36:07.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 194697 5.99 MB/sec execute 588 sec latency 863.345 ms 2024-06-01T02:36:07.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:07 smithi027 ceph-mon[27271]: Monitor daemon marked osd.2 down, but it is still running 2024-06-01T02:36:07.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:07 smithi027 ceph-mon[27271]: map e76 wrongly marked me down at e71 2024-06-01T02:36:07.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:07 smithi027 ceph-mon[27271]: Monitor daemon marked osd.1 down, but it is still running 2024-06-01T02:36:07.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:07 smithi027 ceph-mon[27271]: map e77 wrongly marked me down at e73 2024-06-01T02:36:07.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:07 smithi027 ceph-mon[27271]: osdmap e80: 8 total, 8 up, 8 in 2024-06-01T02:36:07.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:07 smithi027 ceph-mon[27271]: MDS health message cleared (mds.?): 7 slow metadata IOs are blocked > 30 secs, oldest blocked for 73 secs 2024-06-01T02:36:07.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:07 smithi027 ceph-mon[27271]: pgmap v604: 97 pgs: 1 activating+degraded, 16 peering, 26 active+clean, 9 undersized+peered, 10 active+undersized, 17 undersized+degraded+peered, 18 active+undersized+degraded; 346 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 354/1467 objects degraded (24.131%) 2024-06-01T02:36:08.026 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:08 smithi003 ceph-mon[25122]: Monitor daemon marked osd.3 down, but it is still running 2024-06-01T02:36:08.026 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:08 smithi003 ceph-mon[25122]: map e77 wrongly marked me down at e75 2024-06-01T02:36:08.026 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:08 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 354/1467 objects degraded (24.131%), 36 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-06-01T02:36:08.026 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:08 smithi003 ceph-mon[25122]: Health check cleared: MDS_SLOW_METADATA_IO (was: 1 MDSs report slow metadata IOs) 2024-06-01T02:36:08.026 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:08 smithi003 ceph-mon[25122]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:36:08.026 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:08 smithi003 ceph-mon[25122]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:36:08.026 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:08 smithi003 ceph-mon[25122]: osdmap e81: 8 total, 8 up, 8 in 2024-06-01T02:36:08.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 195156 6.00 MB/sec execute 589 sec latency 276.137 ms 2024-06-01T02:36:08.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:08 smithi027 ceph-mon[27271]: Monitor daemon marked osd.3 down, but it is still running 2024-06-01T02:36:08.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:08 smithi027 ceph-mon[27271]: map e77 wrongly marked me down at e75 2024-06-01T02:36:08.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:08 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 354/1467 objects degraded (24.131%), 36 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-06-01T02:36:08.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:08 smithi027 ceph-mon[27271]: Health check cleared: MDS_SLOW_METADATA_IO (was: 1 MDSs report slow metadata IOs) 2024-06-01T02:36:08.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:08 smithi027 ceph-mon[27271]: mds.? [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] up:active 2024-06-01T02:36:08.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:08 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:36:08.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:08 smithi027 ceph-mon[27271]: osdmap e81: 8 total, 8 up, 8 in 2024-06-01T02:36:09.219 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:09 smithi003 ceph-mon[25122]: osdmap e82: 8 total, 8 up, 8 in 2024-06-01T02:36:09.219 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:09 smithi003 ceph-mon[25122]: pgmap v607: 97 pgs: 3 active+recovery_wait+degraded, 1 activating+degraded, 17 peering, 59 active+clean, 3 undersized+peered, 1 active+undersized, 6 undersized+degraded+peered, 7 active+undersized+degraded; 348 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 806 KiB/s wr, 130 op/s; 127/1494 objects degraded (8.501%); 229 KiB/s, 20 keys/s, 11 objects/s recovering 2024-06-01T02:36:09.219 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:09 smithi003 ceph-mon[25122]: Health check update: 27 slow ops, oldest one blocked for 136 sec, daemons [osd.1,osd.2] have slow ops. (SLOW_OPS) 2024-06-01T02:36:09.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 195299 5.99 MB/sec execute 590 sec latency 1020.176 ms 2024-06-01T02:36:09.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:09 smithi027 ceph-mon[27271]: osdmap e82: 8 total, 8 up, 8 in 2024-06-01T02:36:09.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:09 smithi027 ceph-mon[27271]: pgmap v607: 97 pgs: 3 active+recovery_wait+degraded, 1 activating+degraded, 17 peering, 59 active+clean, 3 undersized+peered, 1 active+undersized, 6 undersized+degraded+peered, 7 active+undersized+degraded; 348 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 806 KiB/s wr, 130 op/s; 127/1494 objects degraded (8.501%); 229 KiB/s, 20 keys/s, 11 objects/s recovering 2024-06-01T02:36:09.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:09 smithi027 ceph-mon[27271]: Health check update: 27 slow ops, oldest one blocked for 136 sec, daemons [osd.1,osd.2] have slow ops. (SLOW_OPS) 2024-06-01T02:36:10.244 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:10 smithi003 ceph-mon[25122]: Health check cleared: SLOW_OPS (was: 27 slow ops, oldest one blocked for 136 sec, daemons [osd.1,osd.2] have slow ops.) 2024-06-01T02:36:10.244 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:10 smithi003 ceph-mon[25122]: osdmap e83: 8 total, 8 up, 8 in 2024-06-01T02:36:10.244 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:36:10.292 INFO:teuthology.orchestra.run.smithi003.stdout: 5 195675 6.00 MB/sec execute 591 sec latency 44.703 ms 2024-06-01T02:36:10.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:10 smithi027 ceph-mon[27271]: Health check cleared: SLOW_OPS (was: 27 slow ops, oldest one blocked for 136 sec, daemons [osd.1,osd.2] have slow ops.) 2024-06-01T02:36:10.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:10 smithi027 ceph-mon[27271]: osdmap e83: 8 total, 8 up, 8 in 2024-06-01T02:36:10.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:36:11.254 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:11 smithi003 ceph-mon[25122]: pgmap v609: 97 pgs: 3 active+recovery_wait+degraded, 1 activating+degraded, 1 peering, 91 active+clean, 1 undersized+degraded+peered; 350 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 2.0 MiB/s wr, 217 op/s; 23/1572 objects degraded (1.463%); 1.1 MiB/s, 17 keys/s, 21 objects/s recovering 2024-06-01T02:36:11.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 196174 6.01 MB/sec execute 592 sec latency 27.819 ms 2024-06-01T02:36:11.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:11 smithi027 ceph-mon[27271]: pgmap v609: 97 pgs: 3 active+recovery_wait+degraded, 1 activating+degraded, 1 peering, 91 active+clean, 1 undersized+degraded+peered; 350 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 2.0 MiB/s wr, 217 op/s; 23/1572 objects degraded (1.463%); 1.1 MiB/s, 17 keys/s, 21 objects/s recovering 2024-06-01T02:36:12.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:12 smithi003 ceph-mon[25122]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-06-01T02:36:12.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 196553 6.02 MB/sec execute 593 sec latency 42.270 ms 2024-06-01T02:36:12.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:12 smithi027 ceph-mon[27271]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2024-06-01T02:36:13.282 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:13 smithi003 ceph-mon[25122]: pgmap v610: 97 pgs: 1 active+recovery_wait+degraded, 1 peering, 95 active+clean; 356 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 26 KiB/s rd, 2.8 MiB/s wr, 215 op/s; 6/1677 objects degraded (0.358%); 1007 KiB/s, 13 keys/s, 17 objects/s recovering 2024-06-01T02:36:13.294 INFO:teuthology.orchestra.run.smithi003.stdout: 5 196900 6.02 MB/sec execute 594 sec latency 42.144 ms 2024-06-01T02:36:13.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:13 smithi027 ceph-mon[27271]: pgmap v610: 97 pgs: 1 active+recovery_wait+degraded, 1 peering, 95 active+clean; 356 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 26 KiB/s rd, 2.8 MiB/s wr, 215 op/s; 6/1677 objects degraded (0.358%); 1007 KiB/s, 13 keys/s, 17 objects/s recovering 2024-06-01T02:36:14.040 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:14 smithi003 ceph-mon[25122]: Health check update: Degraded data redundancy: 6/1677 objects degraded (0.358%), 1 pg degraded (PG_DEGRADED) 2024-06-01T02:36:14.040 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:14 smithi003 ceph-mon[25122]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-06-01T02:36:14.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 197300 6.02 MB/sec execute 595 sec latency 42.974 ms 2024-06-01T02:36:14.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:14 smithi027 ceph-mon[27271]: Health check update: Degraded data redundancy: 6/1677 objects degraded (0.358%), 1 pg degraded (PG_DEGRADED) 2024-06-01T02:36:14.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:14 smithi027 ceph-mon[27271]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-06-01T02:36:15.055 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:15 smithi003 ceph-mon[25122]: pgmap v611: 97 pgs: 97 active+clean; 357 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 21 KiB/s rd, 3.1 MiB/s wr, 256 op/s; 792 KiB/s, 10 keys/s, 14 objects/s recovering 2024-06-01T02:36:15.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 197702 6.03 MB/sec execute 596 sec latency 30.354 ms 2024-06-01T02:36:15.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:15 smithi027 ceph-mon[27271]: pgmap v611: 97 pgs: 97 active+clean; 357 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 21 KiB/s rd, 3.1 MiB/s wr, 256 op/s; 792 KiB/s, 10 keys/s, 14 objects/s recovering 2024-06-01T02:36:16.065 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:16 smithi003 ceph-mon[25122]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/1677 objects degraded (0.358%), 1 pg degraded) 2024-06-01T02:36:16.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 198015 6.04 MB/sec execute 597 sec latency 38.319 ms 2024-06-01T02:36:16.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:16 smithi027 ceph-mon[27271]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/1677 objects degraded (0.358%), 1 pg degraded) 2024-06-01T02:36:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:17 smithi003 ceph-mon[25122]: pgmap v612: 97 pgs: 97 active+clean; 359 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 4.6 MiB/s wr, 471 op/s; 594 KiB/s, 9 objects/s recovering 2024-06-01T02:36:17.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 198420 6.05 MB/sec execute 598 sec latency 56.292 ms 2024-06-01T02:36:17.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:17 smithi027 ceph-mon[27271]: pgmap v612: 97 pgs: 97 active+clean; 359 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 4.6 MiB/s wr, 471 op/s; 594 KiB/s, 9 objects/s recovering 2024-06-01T02:36:18.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 198848 6.06 MB/sec execute 599 sec latency 41.829 ms 2024-06-01T02:36:18.867 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:18 smithi003 ceph-mon[25122]: pgmap v613: 97 pgs: 97 active+clean; 352 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 4.2 MiB/s wr, 412 op/s; 513 KiB/s, 6 objects/s recovering 2024-06-01T02:36:19.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:18 smithi027 ceph-mon[27271]: pgmap v613: 97 pgs: 97 active+clean; 352 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 4.2 MiB/s wr, 412 op/s; 513 KiB/s, 6 objects/s recovering 2024-06-01T02:36:19.293 INFO:teuthology.orchestra.run.smithi003.stdout: 5 cleanup 600 sec 2024-06-01T02:36:19.497 INFO:teuthology.orchestra.run.smithi003.stdout: 0 cleanup 600 sec 2024-06-01T02:36:19.497 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:36:19.497 INFO:teuthology.orchestra.run.smithi003.stdout: Operation Count AvgLat MaxLat 2024-06-01T02:36:19.497 INFO:teuthology.orchestra.run.smithi003.stdout: ---------------------------------------- 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: NTCreateX 117040 6.240 26306.242 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: Close 85908 0.126 4.220 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: Rename 4952 9.464 409.091 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: Unlink 23669 8.190 2910.494 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: Qpathinfo 106120 1.469 2905.524 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: Qfileinfo 18466 0.007 1.741 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: Qfsinfo 19415 0.517 8.259 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: Sfileinfo 9538 18.973 24625.113 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: Find 40919 0.903 16.513 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: WriteX 57774 28.214 80184.164 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: ReadX 182891 0.010 3.023 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: LockX 378 0.339 2.589 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: UnlockX 378 0.298 1.452 2024-06-01T02:36:19.498 INFO:teuthology.orchestra.run.smithi003.stdout: Flush 8193 0.003 0.015 2024-06-01T02:36:19.499 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:36:19.499 INFO:teuthology.orchestra.run.smithi003.stdout:Throughput 6.0577 MB/sec 5 clients 5 procs max_latency=80184.171 ms 2024-06-01T02:36:19.500 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-06-01T02:36:19.629 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-06-01T02:36:19.706 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-06-01T02:36:19.732 INFO:teuthology.orchestra.run.smithi003.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-06-01T02:36:19.732 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:36:19.732 INFO:teuthology.orchestra.run.smithi003.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-06-01T02:36:19.733 INFO:teuthology.orchestra.run.smithi003.stdout:4 of 5 processes prepared for launch 0 sec 2024-06-01T02:36:19.833 INFO:teuthology.orchestra.run.smithi003.stdout:5 of 5 processes prepared for launch 0 sec 2024-06-01T02:36:19.833 INFO:teuthology.orchestra.run.smithi003.stdout:releasing clients 2024-06-01T02:36:20.911 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:20 smithi003 ceph-mon[25122]: pgmap v614: 97 pgs: 97 active+clean; 366 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 5.8 MiB/s wr, 506 op/s; 441 KiB/s, 5 objects/s recovering 2024-06-01T02:36:21.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:20 smithi027 ceph-mon[27271]: pgmap v614: 97 pgs: 97 active+clean; 366 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 5.8 MiB/s wr, 506 op/s; 441 KiB/s, 5 objects/s recovering 2024-06-01T02:36:21.833 INFO:teuthology.orchestra.run.smithi003.stdout: 5 337 22.11 MB/sec execute 1 sec latency 64.818 ms 2024-06-01T02:36:22.833 INFO:teuthology.orchestra.run.smithi003.stdout: 5 497 23.80 MB/sec execute 2 sec latency 116.992 ms 2024-06-01T02:36:23.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:22 smithi003 ceph-mon[25122]: pgmap v615: 97 pgs: 97 active+clean; 379 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 517 op/s; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:36:23.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:22 smithi027 ceph-mon[27271]: pgmap v615: 97 pgs: 97 active+clean; 379 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 517 op/s; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:36:23.833 INFO:teuthology.orchestra.run.smithi003.stdout: 5 935 23.34 MB/sec execute 3 sec latency 45.324 ms 2024-06-01T02:36:24.834 INFO:teuthology.orchestra.run.smithi003.stdout: 5 1621 22.98 MB/sec execute 4 sec latency 30.461 ms 2024-06-01T02:36:25.124 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:25 smithi003 ceph-mon[25122]: pgmap v616: 97 pgs: 97 active+clean; 375 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.4 MiB/s wr, 530 op/s; 0 B/s, 0 objects/s recovering 2024-06-01T02:36:25.124 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:36:25.124 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:36:25.124 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:36:25.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:25 smithi027 ceph-mon[27271]: pgmap v616: 97 pgs: 97 active+clean; 375 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.4 MiB/s wr, 530 op/s; 0 B/s, 0 objects/s recovering 2024-06-01T02:36:25.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:36:25.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:36:25.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:36:25.834 INFO:teuthology.orchestra.run.smithi003.stdout: 5 cleanup 5 sec 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: 0 cleanup 5 sec 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: Operation Count AvgLat MaxLat 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: ---------------------------------------- 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: NTCreateX 1920 4.140 116.985 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: Close 1680 3.296 98.475 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: Rename 71 4.360 15.414 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: Unlink 178 2.094 6.497 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: Qpathinfo 1619 1.401 27.671 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: Qfileinfo 573 8.855 96.657 2024-06-01T02:36:26.004 INFO:teuthology.orchestra.run.smithi003.stdout: Qfsinfo 203 0.631 5.832 2024-06-01T02:36:26.005 INFO:teuthology.orchestra.run.smithi003.stdout: Sfileinfo 223 6.302 38.499 2024-06-01T02:36:26.005 INFO:teuthology.orchestra.run.smithi003.stdout: Find 546 1.241 4.792 2024-06-01T02:36:26.005 INFO:teuthology.orchestra.run.smithi003.stdout: WriteX 2203 0.024 0.077 2024-06-01T02:36:26.005 INFO:teuthology.orchestra.run.smithi003.stdout: ReadX 1911 0.013 0.627 2024-06-01T02:36:26.005 INFO:teuthology.orchestra.run.smithi003.stdout: Flush 166 7.265 36.056 2024-06-01T02:36:26.005 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:36:26.005 INFO:teuthology.orchestra.run.smithi003.stdout:Throughput 22.9759 MB/sec 5 clients 5 procs max_latency=116.992 ms 2024-06-01T02:36:26.006 DEBUG:teuthology.parallel:result is None 2024-06-01T02:36:27.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:26 smithi003 ceph-mon[25122]: pgmap v617: 97 pgs: 97 active+clean; 359 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 13 MiB/s wr, 648 op/s 2024-06-01T02:36:27.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:26 smithi027 ceph-mon[27271]: pgmap v617: 97 pgs: 97 active+clean; 359 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 13 MiB/s wr, 648 op/s 2024-06-01T02:36:28.968 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:28 smithi003 ceph-mon[25122]: pgmap v618: 97 pgs: 97 active+clean; 341 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 13 MiB/s wr, 543 op/s 2024-06-01T02:36:29.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:28 smithi027 ceph-mon[27271]: pgmap v618: 97 pgs: 97 active+clean; 341 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 13 MiB/s wr, 543 op/s 2024-06-01T02:36:31.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:30 smithi003 ceph-mon[25122]: pgmap v619: 97 pgs: 97 active+clean; 285 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 13 MiB/s wr, 551 op/s 2024-06-01T02:36:31.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:30 smithi027 ceph-mon[27271]: pgmap v619: 97 pgs: 97 active+clean; 285 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 13 MiB/s wr, 551 op/s 2024-06-01T02:36:33.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:32 smithi003 ceph-mon[25122]: pgmap v620: 97 pgs: 97 active+clean; 253 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 417 op/s 2024-06-01T02:36:33.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:32 smithi027 ceph-mon[27271]: pgmap v620: 97 pgs: 97 active+clean; 253 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 417 op/s 2024-06-01T02:36:35.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:35 smithi003 ceph-mon[25122]: pgmap v621: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 8.2 MiB/s wr, 352 op/s 2024-06-01T02:36:35.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:35 smithi027 ceph-mon[27271]: pgmap v621: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 8.2 MiB/s wr, 352 op/s 2024-06-01T02:36:37.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:36 smithi003 ceph-mon[25122]: pgmap v622: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.7 MiB/s wr, 317 op/s 2024-06-01T02:36:37.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:36 smithi027 ceph-mon[27271]: pgmap v622: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.7 MiB/s wr, 317 op/s 2024-06-01T02:36:39.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:38 smithi003 ceph-mon[25122]: pgmap v623: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.7 MiB/s wr, 142 op/s 2024-06-01T02:36:39.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:38 smithi027 ceph-mon[27271]: pgmap v623: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.7 MiB/s wr, 142 op/s 2024-06-01T02:36:41.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:40 smithi003 ceph-mon[25122]: pgmap v624: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 414 KiB/s wr, 53 op/s 2024-06-01T02:36:41.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:40 smithi027 ceph-mon[27271]: pgmap v624: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 414 KiB/s wr, 53 op/s 2024-06-01T02:36:43.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:42 smithi003 ceph-mon[25122]: pgmap v625: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 26 KiB/s wr, 15 op/s 2024-06-01T02:36:43.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:42 smithi027 ceph-mon[27271]: pgmap v625: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 26 KiB/s wr, 15 op/s 2024-06-01T02:36:45.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:45 smithi003 ceph-mon[25122]: pgmap v626: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 3 op/s 2024-06-01T02:36:45.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:45 smithi027 ceph-mon[27271]: pgmap v626: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 3 op/s 2024-06-01T02:36:47.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:46 smithi003 ceph-mon[25122]: pgmap v627: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-06-01T02:36:47.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:46 smithi027 ceph-mon[27271]: pgmap v627: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-06-01T02:36:49.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:48 smithi003 ceph-mon[25122]: pgmap v628: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-06-01T02:36:49.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:48 smithi027 ceph-mon[27271]: pgmap v628: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-06-01T02:36:51.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:50 smithi003 ceph-mon[25122]: pgmap v629: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:36:51.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:50 smithi027 ceph-mon[27271]: pgmap v629: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:36:53.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:52 smithi003 ceph-mon[25122]: pgmap v630: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:36:53.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:52 smithi027 ceph-mon[27271]: pgmap v630: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:36:55.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:55 smithi003 ceph-mon[25122]: pgmap v631: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:36:55.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:55 smithi027 ceph-mon[27271]: pgmap v631: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:36:57.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:56 smithi003 ceph-mon[25122]: pgmap v632: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:36:57.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:56 smithi027 ceph-mon[27271]: pgmap v632: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:36:59.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:36:58 smithi003 ceph-mon[25122]: pgmap v633: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:36:59.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:36:58 smithi027 ceph-mon[27271]: pgmap v633: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:01.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:00 smithi003 ceph-mon[25122]: pgmap v634: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:01.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:00 smithi027 ceph-mon[27271]: pgmap v634: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:03.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:02 smithi003 ceph-mon[25122]: pgmap v635: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:03.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:02 smithi027 ceph-mon[27271]: pgmap v635: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:05.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:05 smithi003 ceph-mon[25122]: pgmap v636: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:05.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:05 smithi027 ceph-mon[27271]: pgmap v636: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:07.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:06 smithi003 ceph-mon[25122]: pgmap v637: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:07.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:06 smithi027 ceph-mon[27271]: pgmap v637: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:09.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:08 smithi003 ceph-mon[25122]: pgmap v638: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:09.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:08 smithi027 ceph-mon[27271]: pgmap v638: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:11.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:10 smithi003 ceph-mon[25122]: pgmap v639: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:11.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:10 smithi003 ceph-mon[25122]: Health check cleared: OSD_SLOW_PING_TIME_BACK (was: Slow OSD heartbeats on back (longest 37968.531ms)) 2024-06-01T02:37:11.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:10 smithi003 ceph-mon[25122]: Health check cleared: OSD_SLOW_PING_TIME_FRONT (was: Slow OSD heartbeats on front (longest 37875.098ms)) 2024-06-01T02:37:11.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:10 smithi027 ceph-mon[27271]: pgmap v639: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:11.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:10 smithi027 ceph-mon[27271]: Health check cleared: OSD_SLOW_PING_TIME_BACK (was: Slow OSD heartbeats on back (longest 37968.531ms)) 2024-06-01T02:37:11.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:10 smithi027 ceph-mon[27271]: Health check cleared: OSD_SLOW_PING_TIME_FRONT (was: Slow OSD heartbeats on front (longest 37875.098ms)) 2024-06-01T02:37:13.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:12 smithi003 ceph-mon[25122]: pgmap v640: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:13.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:12 smithi027 ceph-mon[27271]: pgmap v640: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:15.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:15 smithi003 ceph-mon[25122]: pgmap v641: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:15.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:15 smithi027 ceph-mon[27271]: pgmap v641: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:17.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:16 smithi003 ceph-mon[25122]: pgmap v642: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:17.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:16 smithi027 ceph-mon[27271]: pgmap v642: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:19.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:18 smithi003 ceph-mon[25122]: pgmap v643: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:19.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:18 smithi027 ceph-mon[27271]: pgmap v643: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:21.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:20 smithi003 ceph-mon[25122]: pgmap v644: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:21.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:20 smithi027 ceph-mon[27271]: pgmap v644: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:23.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:22 smithi003 ceph-mon[25122]: pgmap v645: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:23.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:22 smithi027 ceph-mon[27271]: pgmap v645: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:25 smithi003 ceph-mon[25122]: pgmap v646: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:25.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:37:25.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:37:25.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:37:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:25 smithi027 ceph-mon[27271]: pgmap v646: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:37:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:37:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:37:27.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:26 smithi003 ceph-mon[25122]: pgmap v647: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:27.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:26 smithi027 ceph-mon[27271]: pgmap v647: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:29.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:28 smithi003 ceph-mon[25122]: pgmap v648: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:29.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:28 smithi027 ceph-mon[27271]: pgmap v648: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:31.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:30 smithi003 ceph-mon[25122]: pgmap v649: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:31.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:30 smithi027 ceph-mon[27271]: pgmap v649: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:33.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:32 smithi003 ceph-mon[25122]: pgmap v650: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:33.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:32 smithi027 ceph-mon[27271]: pgmap v650: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:35.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:35 smithi003 ceph-mon[25122]: pgmap v651: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:35.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:35 smithi027 ceph-mon[27271]: pgmap v651: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:37.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:36 smithi003 ceph-mon[25122]: pgmap v652: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:37.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:36 smithi027 ceph-mon[27271]: pgmap v652: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:39.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:38 smithi003 ceph-mon[25122]: pgmap v653: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:39.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:38 smithi027 ceph-mon[27271]: pgmap v653: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:41.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:40 smithi003 ceph-mon[25122]: pgmap v654: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:41.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:40 smithi027 ceph-mon[27271]: pgmap v654: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:43.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:42 smithi003 ceph-mon[25122]: pgmap v655: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:43.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:42 smithi027 ceph-mon[27271]: pgmap v655: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:45.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:45 smithi003 ceph-mon[25122]: pgmap v656: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:45.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:45 smithi027 ceph-mon[27271]: pgmap v656: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:47.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:46 smithi003 ceph-mon[25122]: pgmap v657: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:47.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:46 smithi027 ceph-mon[27271]: pgmap v657: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:49.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:48 smithi003 ceph-mon[25122]: pgmap v658: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:49.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:48 smithi027 ceph-mon[27271]: pgmap v658: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:51.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:50 smithi003 ceph-mon[25122]: pgmap v659: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:51.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:50 smithi027 ceph-mon[27271]: pgmap v659: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:37:53.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:52 smithi003 ceph-mon[25122]: pgmap v660: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:53.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:52 smithi027 ceph-mon[27271]: pgmap v660: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:55.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:55 smithi003 ceph-mon[25122]: pgmap v661: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:55.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:55 smithi027 ceph-mon[27271]: pgmap v661: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:57.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:56 smithi003 ceph-mon[25122]: pgmap v662: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:57.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:56 smithi027 ceph-mon[27271]: pgmap v662: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:59.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:37:58 smithi003 ceph-mon[25122]: pgmap v663: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:37:59.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:37:58 smithi027 ceph-mon[27271]: pgmap v663: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:01.085 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:00 smithi003 ceph-mon[25122]: pgmap v664: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:01.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:00 smithi027 ceph-mon[27271]: pgmap v664: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:02.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:02 smithi003 ceph-mon[25122]: pgmap v665: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 190 B/s rd, 0 op/s 2024-06-01T02:38:02.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:02 smithi003 ceph-mon[25122]: pgmap v666: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:02.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:38:02.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:02 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:38:02.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:02 smithi027 ceph-mon[27271]: pgmap v665: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 190 B/s rd, 0 op/s 2024-06-01T02:38:02.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:02 smithi027 ceph-mon[27271]: pgmap v666: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:02.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:38:02.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:02 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:38:04.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:04 smithi003 ceph-mon[25122]: pgmap v667: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:04.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:04 smithi027 ceph-mon[27271]: pgmap v667: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:06.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:06 smithi003 ceph-mon[25122]: pgmap v668: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:06.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:06 smithi027 ceph-mon[27271]: pgmap v668: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:08.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:08 smithi003 ceph-mon[25122]: pgmap v669: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:08.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:08 smithi027 ceph-mon[27271]: pgmap v669: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:10.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:10 smithi003 ceph-mon[25122]: pgmap v670: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:10.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:10 smithi027 ceph-mon[27271]: pgmap v670: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 233 B/s rd, 0 op/s 2024-06-01T02:38:12.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:12 smithi003 ceph-mon[25122]: pgmap v671: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:38:12.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:12 smithi027 ceph-mon[27271]: pgmap v671: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:38:14.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:14 smithi003 ceph-mon[25122]: pgmap v672: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:14.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:14 smithi027 ceph-mon[27271]: pgmap v672: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:16.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:16 smithi003 ceph-mon[25122]: pgmap v673: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:16.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:16 smithi027 ceph-mon[27271]: pgmap v673: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:18.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:18 smithi003 ceph-mon[25122]: pgmap v674: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:18.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:18 smithi027 ceph-mon[27271]: pgmap v674: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:20.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:20 smithi003 ceph-mon[25122]: pgmap v675: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:20.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:20 smithi027 ceph-mon[27271]: pgmap v675: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:22.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:22 smithi003 ceph-mon[25122]: pgmap v676: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:22.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:22 smithi027 ceph-mon[27271]: pgmap v676: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:24.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:24 smithi003 ceph-mon[25122]: pgmap v677: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:24.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:24 smithi027 ceph-mon[27271]: pgmap v677: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:25.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:38:25.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:38:25.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:38:25.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:38:26.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:26 smithi003 ceph-mon[25122]: pgmap v678: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:26.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:26 smithi027 ceph-mon[27271]: pgmap v678: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:28.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:28 smithi003 ceph-mon[25122]: pgmap v679: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:28.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:28 smithi027 ceph-mon[27271]: pgmap v679: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:30.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:30 smithi003 ceph-mon[25122]: pgmap v680: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:30.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:30 smithi027 ceph-mon[27271]: pgmap v680: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:32.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:32 smithi003 ceph-mon[25122]: pgmap v681: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:32.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:32 smithi027 ceph-mon[27271]: pgmap v681: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:34.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:34 smithi003 ceph-mon[25122]: pgmap v682: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:34.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:34 smithi027 ceph-mon[27271]: pgmap v682: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:36.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:36 smithi003 ceph-mon[25122]: pgmap v683: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:36.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:36 smithi027 ceph-mon[27271]: pgmap v683: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:38.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:38 smithi003 ceph-mon[25122]: pgmap v684: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:38.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:38 smithi027 ceph-mon[27271]: pgmap v684: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:39.221 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-06-01T02:38:40.161 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:40 smithi003 ceph-mon[25122]: pgmap v685: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:40.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:40 smithi027 ceph-mon[27271]: pgmap v685: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:42.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:42 smithi003 ceph-mon[25122]: pgmap v686: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:42.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:42 smithi027 ceph-mon[27271]: pgmap v686: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:43.418 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph config set global log_to_journald false --force' 2024-06-01T02:38:44.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:44 smithi003 ceph-mon[25122]: pgmap v687: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:44.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:44 smithi027 ceph-mon[27271]: pgmap v687: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:46.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:46 smithi003 ceph-mon[25122]: pgmap v688: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:46 smithi027 ceph-mon[27271]: pgmap v688: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:47.223 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph mgr module enable nfs --force' 2024-06-01T02:38:48.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:48 smithi003 ceph-mon[25122]: pgmap v689: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:48.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:48 smithi027 ceph-mon[27271]: pgmap v689: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:50.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:50 smithi003 ceph-mon[25122]: pgmap v690: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:50.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:50 smithi027 ceph-mon[27271]: pgmap v690: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:38:51.066 INFO:teuthology.orchestra.run.smithi003.stderr:module 'nfs' is already enabled 2024-06-01T02:38:51.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:51 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/4261562361' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-06-01T02:38:51.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:51 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/4261562361' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-06-01T02:38:52.018 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-06-01T02:38:52.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:52 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/4261562361' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-06-01T02:38:52.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:52 smithi003 ceph-mon[25122]: mgrmap e21: smithi003.njsqxz(active, since 20m), standbys: smithi027.tujskj 2024-06-01T02:38:52.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:52 smithi003 ceph-mon[25122]: pgmap v691: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:52.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:52 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/4261562361' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-06-01T02:38:52.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:52 smithi027 ceph-mon[27271]: mgrmap e21: smithi003.njsqxz(active, since 20m), standbys: smithi027.tujskj 2024-06-01T02:38:52.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:52 smithi027 ceph-mon[27271]: pgmap v691: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:54.730 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:54 smithi003 ceph-mon[25122]: pgmap v692: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:54.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:54 smithi027 ceph-mon[27271]: pgmap v692: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:55.047 INFO:teuthology.orchestra.run.smithi003.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba 2024-06-01T02:38:56.091 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-06-01T02:38:56.092 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:38:56.092 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-06-01T02:38:56.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:56 smithi003 ceph-mon[25122]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:38:56.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:56 smithi003 ceph-mon[25122]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba 2024-06-01T02:38:56.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:56 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:38:56.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:56 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:38:56.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:56 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:38:56.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:56 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:38:56.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:56 smithi003 ceph-mon[25122]: pgmap v693: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:56.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:56 smithi027 ceph-mon[27271]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:38:56.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:56 smithi027 ceph-mon[27271]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba 2024-06-01T02:38:56.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:56 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:38:56.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:56 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:38:56.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:56 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:38:56.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:56 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:38:56.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:56 smithi027 ceph-mon[27271]: pgmap v693: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:38:58 smithi003 ceph-mon[25122]: pgmap v694: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:38:58.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:38:58 smithi027 ceph-mon[27271]: pgmap v694: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:00.290 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:39:00.291 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:00 smithi003 ceph-mon[25122]: pgmap v695: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:00.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:00 smithi027 ceph-mon[27271]: pgmap v695: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:00.909 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:39:00.909 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (19m) 14m ago 21m 24.2M - ba2b418f427c ceda73487cd9 2024-06-01T02:39:00.909 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (21m) 14m ago 21m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:39:00.909 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (20m) 3m ago 20m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:39:00.909 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (19m) 14m ago 20m 52.0M - 8.3.5 dad864ee21e9 32cd8820aa5e 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (15m) 14m ago 15m 12.6M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (15m) 3m ago 15m 15.9M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:9283 running (22m) 14m ago 22m 462M - 17.2.0 e1d6a67b021e ac6fbe8a9d2d 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283 running (20m) 3m ago 20m 417M - 17.2.0 e1d6a67b021e e778130b1363 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (22m) 14m ago 22m 46.2M 2048M 17.2.0 e1d6a67b021e 108dda5fe7cc 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (20m) 3m ago 20m 42.9M 2048M 17.2.0 e1d6a67b021e edc537c54a3a 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (14m) 14m ago 14m 49.1M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (14m) 3m ago 14m 86.7M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (20m) 14m ago 20m 18.1M - 1dbe0e931976 6338b03aebaf 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (20m) 3m ago 20m 17.7M - 1dbe0e931976 3ee520f9eebb 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (19m) 14m ago 19m 42.8M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:39:00.910 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (19m) 14m ago 19m 43.7M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:39:00.911 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (18m) 14m ago 18m 39.3M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:39:00.911 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (18m) 14m ago 18m 39.2M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:39:00.911 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (17m) 3m ago 17m 568M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:39:00.911 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (17m) 3m ago 17m 443M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:39:00.911 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (17m) 3m ago 17m 342M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:39:00.911 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (16m) 3m ago 16m 580M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:39:00.911 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (19m) 14m ago 19m 62.1M - 514e6a882f6e 70b2c7c41ae9 2024-06-01T02:39:01.303 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:39:01.304 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:39:01.305 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-06-01T02:39:01.305 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:39:01.305 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:39:01.305 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:01 smithi003 ceph-mon[25122]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:01.305 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:01 smithi003 ceph-mon[25122]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:01 smithi027 ceph-mon[27271]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:01 smithi027 ceph-mon[27271]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:01.618 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:39:01.618 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba", 2024-06-01T02:39:01.618 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:39:01.618 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [], 2024-06-01T02:39:01.618 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "", 2024-06-01T02:39:01.618 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "" 2024-06-01T02:39:01.618 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_WARN failed to probe daemons or devices 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout: host smithi003 `cephadm ls` failed: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /bin/podman version --format {{.Client.Version}} 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout:/bin/podman: stderr Error: beginning transaction: database is locked 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout:Traceback (most recent call last): 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8634, in 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout: main() 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8620, in main 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout: check_container_engine(ctx) 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 2165, in check_container_engine 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout: engine.get_version(ctx) 2024-06-01T02:39:01.987 INFO:teuthology.orchestra.run.smithi003.stdout: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 195, in get_version 2024-06-01T02:39:01.988 INFO:teuthology.orchestra.run.smithi003.stdout: out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}']) 2024-06-01T02:39:01.988 INFO:teuthology.orchestra.run.smithi003.stdout: File "/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 1615, in call_throws 2024-06-01T02:39:01.988 INFO:teuthology.orchestra.run.smithi003.stdout: raise RuntimeError(f'Failed command: {" ".join(command)}: {s}') 2024-06-01T02:39:01.988 INFO:teuthology.orchestra.run.smithi003.stdout:RuntimeError: Failed command: /bin/podman version --format {{.Client.Version}}: Error: beginning transaction: database is locked 2024-06-01T02:39:02.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:02 smithi003 ceph-mon[25122]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:02.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:02 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3201162626' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:39:02.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:02 smithi003 ceph-mon[25122]: pgmap v696: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:02.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:02 smithi003 ceph-mon[25122]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:02.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:02 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3817579453' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:39:02.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:02 smithi027 ceph-mon[27271]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:02.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:02 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/3201162626' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:39:02.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:02 smithi027 ceph-mon[27271]: pgmap v696: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:02.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:02 smithi027 ceph-mon[27271]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:02.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:02 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/3817579453' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:39:04.835 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:04 smithi003 ceph-mon[25122]: pgmap v697: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:04.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:04 smithi027 ceph-mon[27271]: pgmap v697: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:06.729 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:06 smithi003 ceph-mon[25122]: pgmap v698: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:06.729 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:06 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:39:06.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:06 smithi027 ceph-mon[27271]: pgmap v698: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:06.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:06 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:39:08.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:07 smithi003 ceph-mon[25122]: pgmap v699: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:08.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:07 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:39:08.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:07 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:39:08.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:07 smithi003 ceph-mon[25122]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba 2024-06-01T02:39:08.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:07 smithi003 ceph-mon[25122]: Health check cleared: CEPHADM_REFRESH_FAILED (was: failed to probe daemons or devices) 2024-06-01T02:39:08.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:07 smithi003 ceph-mon[25122]: Cluster is now healthy 2024-06-01T02:39:08.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:07 smithi027 ceph-mon[27271]: pgmap v699: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:08.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:07 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:39:08.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:07 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:39:08.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:07 smithi027 ceph-mon[27271]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba 2024-06-01T02:39:08.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:07 smithi027 ceph-mon[27271]: Health check cleared: CEPHADM_REFRESH_FAILED (was: failed to probe daemons or devices) 2024-06-01T02:39:08.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:07 smithi027 ceph-mon[27271]: Cluster is now healthy 2024-06-01T02:39:10.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:10 smithi003 ceph-mon[25122]: pgmap v700: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:10.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:10 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:39:10.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:10 smithi027 ceph-mon[27271]: pgmap v700: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:10.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:10 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:39:12.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:12 smithi003 ceph-mon[25122]: pgmap v701: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:12.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:12 smithi027 ceph-mon[27271]: pgmap v701: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:14.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:14 smithi003 ceph-mon[25122]: pgmap v702: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:14.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:14 smithi027 ceph-mon[27271]: pgmap v702: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:16.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:16 smithi003 ceph-mon[25122]: pgmap v703: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:16.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:16 smithi027 ceph-mon[27271]: pgmap v703: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:18.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:18 smithi003 ceph-mon[25122]: pgmap v704: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:18.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:18 smithi027 ceph-mon[27271]: pgmap v704: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-06-01T02:39:20.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:20 smithi003 ceph-mon[25122]: pgmap v705: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:20.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:20 smithi027 ceph-mon[27271]: pgmap v705: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:22.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:22 smithi003 ceph-mon[25122]: pgmap v706: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:22.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:22 smithi027 ceph-mon[27271]: pgmap v706: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:24.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:24 smithi003 ceph-mon[25122]: pgmap v707: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:24.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:24 smithi027 ceph-mon[27271]: pgmap v707: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:25.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:39:25.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:39:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:39:25.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:39:26.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:26 smithi003 ceph-mon[25122]: pgmap v708: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:26.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:26 smithi027 ceph-mon[27271]: pgmap v708: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:28.335 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:28 smithi003 ceph-mon[25122]: pgmap v709: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:28.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:28 smithi027 ceph-mon[27271]: pgmap v709: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:30.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:30 smithi003 ceph-mon[25122]: pgmap v710: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:30.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:30 smithi027 ceph-mon[27271]: pgmap v710: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:32.334 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:39:32.334 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:32 smithi003 ceph-mon[25122]: pgmap v711: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:32.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:32 smithi027 ceph-mon[27271]: pgmap v711: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:32.974 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:39:32.974 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (20m) 26s ago 21m 26.0M - ba2b418f427c ceda73487cd9 2024-06-01T02:39:32.974 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (21m) 26s ago 21m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:39:32.974 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (21m) 4m ago 21m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:39:32.974 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (19m) 26s ago 21m 52.9M - 8.3.5 dad864ee21e9 32cd8820aa5e 2024-06-01T02:39:32.974 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (15m) 26s ago 15m 40.0M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:39:32.974 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (15m) 4m ago 15m 15.9M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:39:32.974 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:9283 running (22m) 26s ago 22m 471M - 17.2.0 e1d6a67b021e ac6fbe8a9d2d 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283 running (20m) 4m ago 20m 417M - 17.2.0 e1d6a67b021e e778130b1363 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (22m) 26s ago 22m 60.6M 2048M 17.2.0 e1d6a67b021e 108dda5fe7cc 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (20m) 4m ago 20m 42.9M 2048M 17.2.0 e1d6a67b021e edc537c54a3a 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (15m) 26s ago 15m 249M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (15m) 4m ago 15m 86.7M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (21m) 26s ago 21m 17.3M - 1dbe0e931976 6338b03aebaf 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (20m) 4m ago 20m 17.7M - 1dbe0e931976 3ee520f9eebb 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (20m) 26s ago 20m 439M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (19m) 26s ago 19m 358M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (19m) 26s ago 19m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (18m) 26s ago 18m 369M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (18m) 4m ago 18m 568M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (18m) 4m ago 18m 443M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (17m) 4m ago 17m 342M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:39:32.975 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (17m) 4m ago 17m 580M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:39:32.976 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (20m) 26s ago 20m 66.6M - 514e6a882f6e 70b2c7c41ae9 2024-06-01T02:39:33.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:33 smithi003 ceph-mon[25122]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:33.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:33 smithi003 ceph-mon[25122]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:33.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:33 smithi027 ceph-mon[27271]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:33.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:33 smithi027 ceph-mon[27271]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:33.366 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:39:33.366 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:39:33.366 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:39:33.367 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-06-01T02:39:33.368 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:39:33.368 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:39:33.706 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:39:33.706 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba", 2024-06-01T02:39:33.707 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:39:33.707 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [], 2024-06-01T02:39:33.707 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "", 2024-06-01T02:39:33.707 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba image" 2024-06-01T02:39:33.707 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:39:34.103 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:39:34.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:34 smithi027 ceph-mon[27271]: pgmap v712: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:34.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:34 smithi027 ceph-mon[27271]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:34.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:34 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/875940547' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:39:34.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:34 smithi027 ceph-mon[27271]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:34 smithi003 ceph-mon[25122]: pgmap v712: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:34 smithi003 ceph-mon[25122]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:34 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/875940547' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:39:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:34 smithi003 ceph-mon[25122]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:39:35.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:35 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/3491645392' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:39:35.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:35 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3491645392' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:39:36.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:36 smithi027 ceph-mon[27271]: pgmap v713: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:36.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:36 smithi003 ceph-mon[25122]: pgmap v713: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:38.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:38 smithi027 ceph-mon[27271]: pgmap v714: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:38.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:38 smithi003 ceph-mon[25122]: pgmap v714: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:40.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:40 smithi027 ceph-mon[27271]: pgmap v715: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:40.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:40 smithi003 ceph-mon[25122]: pgmap v715: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:42.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:42 smithi003 ceph-mon[25122]: pgmap v716: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:42.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:42 smithi027 ceph-mon[27271]: pgmap v716: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:44.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:44 smithi003 ceph-mon[25122]: pgmap v717: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:44.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:44 smithi027 ceph-mon[27271]: pgmap v717: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:46.404 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:46 smithi027 ceph-mon[27271]: pgmap v718: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:46.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:46 smithi003 ceph-mon[25122]: pgmap v718: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:48.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:48 smithi003 ceph-mon[25122]: pgmap v719: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:48.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:48 smithi027 ceph-mon[27271]: pgmap v719: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:50.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:50 smithi003 ceph-mon[25122]: pgmap v720: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:50.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:50 smithi027 ceph-mon[27271]: pgmap v720: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:52.585 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:52 smithi003 ceph-mon[25122]: pgmap v721: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:52.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:52 smithi027 ceph-mon[27271]: pgmap v721: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:39:54.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:54 smithi003 ceph-mon[25122]: pgmap v722: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:54.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:54 smithi027 ceph-mon[27271]: pgmap v722: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:56.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:56 smithi003 ceph-mon[25122]: pgmap v723: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:56.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:56 smithi027 ceph-mon[27271]: pgmap v723: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:58.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:39:58 smithi003 ceph-mon[25122]: pgmap v724: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:39:58.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:39:58 smithi027 ceph-mon[27271]: pgmap v724: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:00.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:00 smithi003 ceph-mon[25122]: pgmap v725: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:00.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:00 smithi003 ceph-mon[25122]: overall HEALTH_OK 2024-06-01T02:40:00.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:00 smithi027 ceph-mon[27271]: pgmap v725: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:00.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:00 smithi027 ceph-mon[27271]: overall HEALTH_OK 2024-06-01T02:40:02.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:02 smithi003 ceph-mon[25122]: pgmap v726: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:02.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:02 smithi027 ceph-mon[27271]: pgmap v726: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:04.458 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:40:04.459 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:04 smithi003 ceph-mon[25122]: pgmap v727: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:04.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:04 smithi027 ceph-mon[27271]: pgmap v727: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:05.114 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (20m) 58s ago 22m 26.0M - ba2b418f427c ceda73487cd9 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (22m) 58s ago 22m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (21m) 5m ago 21m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (20m) 58s ago 21m 52.9M - 8.3.5 dad864ee21e9 32cd8820aa5e 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (16m) 58s ago 16m 40.0M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (16m) 5m ago 16m 15.9M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:9283 running (23m) 58s ago 23m 471M - 17.2.0 e1d6a67b021e ac6fbe8a9d2d 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283 running (21m) 5m ago 21m 417M - 17.2.0 e1d6a67b021e e778130b1363 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (23m) 58s ago 23m 60.6M 2048M 17.2.0 e1d6a67b021e 108dda5fe7cc 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (21m) 5m ago 21m 42.9M 2048M 17.2.0 e1d6a67b021e edc537c54a3a 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (15m) 58s ago 15m 249M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (15m) 5m ago 15m 86.7M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:40:05.115 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (21m) 58s ago 21m 17.3M - 1dbe0e931976 6338b03aebaf 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (21m) 5m ago 21m 17.7M - 1dbe0e931976 3ee520f9eebb 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (20m) 58s ago 20m 439M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (20m) 58s ago 20m 358M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (19m) 58s ago 19m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (19m) 58s ago 19m 369M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (18m) 5m ago 18m 568M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (18m) 5m ago 18m 443M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (18m) 5m ago 18m 342M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (17m) 5m ago 17m 580M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:40:05.116 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (21m) 58s ago 21m 66.6M - 514e6a882f6e 70b2c7c41ae9 2024-06-01T02:40:05.508 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:40:05.509 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:40:05.510 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:40:05.510 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:40:05.510 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:40:05.510 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-06-01T02:40:05.510 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:40:05.510 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:40:05.510 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:05 smithi003 ceph-mon[25122]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:05.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:05 smithi027 ceph-mon[27271]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:06.010 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:40:06.011 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba", 2024-06-01T02:40:06.011 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:40:06.011 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [], 2024-06-01T02:40:06.011 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "", 2024-06-01T02:40:06.011 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:3d89accfe3201286f000e065a1faea2a88c7a6ba image" 2024-06-01T02:40:06.011 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:40:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:06 smithi003 ceph-mon[25122]: pgmap v728: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:06 smithi003 ceph-mon[25122]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:06 smithi003 ceph-mon[25122]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:06 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/3108297893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:40:06.461 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:40:06.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:06 smithi027 ceph-mon[27271]: pgmap v728: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:06.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:06 smithi027 ceph-mon[27271]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:06.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:06 smithi027 ceph-mon[27271]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:06.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:06 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/3108297893' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:40:07.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:07 smithi003 ceph-mon[25122]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:07.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:07 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/2266855971' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:40:07.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:07 smithi027 ceph-mon[27271]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:07.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:07 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/2266855971' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:40:08.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:08 smithi003 ceph-mon[25122]: pgmap v729: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:08.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:08 smithi027 ceph-mon[27271]: pgmap v729: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:10.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:10 smithi003 ceph-mon[25122]: pgmap v730: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:10.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:10 smithi027 ceph-mon[27271]: pgmap v730: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:12.535 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:12 smithi003 ceph-mon[25122]: pgmap v731: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:12.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:12 smithi027 ceph-mon[27271]: pgmap v731: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:13.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:13 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:13.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:13 smithi003 ceph-mon[25122]: Upgrade: Target is version 19.0.0-2946-g3d89accf (unknown) 2024-06-01T02:40:13.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:13 smithi003 ceph-mon[25122]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9'] 2024-06-01T02:40:13.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:13 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:40:13.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:13 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:13.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:13 smithi003 ceph-mon[25122]: Upgrade: Need to upgrade myself (mgr.smithi003.njsqxz) 2024-06-01T02:40:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:13 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:13 smithi027 ceph-mon[27271]: Upgrade: Target is version 19.0.0-2946-g3d89accf (unknown) 2024-06-01T02:40:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:13 smithi027 ceph-mon[27271]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9'] 2024-06-01T02:40:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:13 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:40:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:13 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:13 smithi027 ceph-mon[27271]: Upgrade: Need to upgrade myself (mgr.smithi003.njsqxz) 2024-06-01T02:40:14.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:14 smithi003 ceph-mon[25122]: pgmap v732: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:14.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:14 smithi003 ceph-mon[25122]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 on smithi027 2024-06-01T02:40:14.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:14 smithi027 ceph-mon[27271]: pgmap v732: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:14.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:14 smithi027 ceph-mon[27271]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 on smithi027 2024-06-01T02:40:16.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:16 smithi003 ceph-mon[25122]: pgmap v733: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:16.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:16 smithi027 ceph-mon[27271]: pgmap v733: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:18.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:18 smithi027 ceph-mon[27271]: pgmap v734: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:18.970 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:18 smithi003 ceph-mon[25122]: pgmap v734: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:20.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:20 smithi027 ceph-mon[27271]: pgmap v735: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:21.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:20 smithi003 ceph-mon[25122]: pgmap v735: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:22.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:22 smithi027 ceph-mon[27271]: pgmap v736: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:23.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:22 smithi003 ceph-mon[25122]: pgmap v736: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:24.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:24 smithi027 ceph-mon[27271]: pgmap v737: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:25.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:24 smithi003 ceph-mon[25122]: pgmap v737: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:25.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:40:25.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:25 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:40:26.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:40:26.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:25 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:40:26.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:26 smithi027 ceph-mon[27271]: pgmap v738: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:27.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:26 smithi003 ceph-mon[25122]: pgmap v738: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:28.969 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:28 smithi003 ceph-mon[25122]: pgmap v739: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:29.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:28 smithi027 ceph-mon[27271]: pgmap v739: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:31.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:30 smithi003 ceph-mon[25122]: pgmap v740: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:31.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:30 smithi027 ceph-mon[27271]: pgmap v740: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:33.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:32 smithi003 ceph-mon[25122]: pgmap v741: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:33.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:32 smithi027 ceph-mon[27271]: pgmap v741: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:35.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:34 smithi003 ceph-mon[25122]: pgmap v742: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:35.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:34 smithi027 ceph-mon[27271]: pgmap v742: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:36.803 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:40:37.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:36 smithi003 ceph-mon[25122]: pgmap v743: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:37.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:36 smithi027 ceph-mon[27271]: pgmap v743: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:37.411 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:40:37.411 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (21m) 90s ago 22m 26.0M - ba2b418f427c ceda73487cd9 2024-06-01T02:40:37.411 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (22m) 90s ago 22m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:40:37.411 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (22m) 5m ago 22m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:40:37.411 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (21m) 90s ago 22m 52.9M - 8.3.5 dad864ee21e9 32cd8820aa5e 2024-06-01T02:40:37.411 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (16m) 90s ago 16m 40.0M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (16m) 5m ago 16m 15.9M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:9283 running (23m) 90s ago 23m 471M - 17.2.0 e1d6a67b021e ac6fbe8a9d2d 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283 running (22m) 5m ago 22m 417M - 17.2.0 e1d6a67b021e e778130b1363 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (23m) 90s ago 23m 60.6M 2048M 17.2.0 e1d6a67b021e 108dda5fe7cc 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (21m) 5m ago 21m 42.9M 2048M 17.2.0 e1d6a67b021e edc537c54a3a 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (16m) 90s ago 16m 249M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (16m) 5m ago 16m 86.7M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (22m) 90s ago 22m 17.3M - 1dbe0e931976 6338b03aebaf 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (21m) 5m ago 21m 17.7M - 1dbe0e931976 3ee520f9eebb 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (21m) 90s ago 21m 439M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (20m) 90s ago 20m 358M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (20m) 90s ago 20m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (19m) 90s ago 19m 369M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:40:37.412 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (19m) 5m ago 19m 568M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:40:37.413 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (19m) 5m ago 19m 443M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:40:37.413 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (18m) 5m ago 18m 342M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:40:37.413 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (18m) 5m ago 18m 580M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:40:37.413 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (21m) 90s ago 21m 66.6M - 514e6a882f6e 70b2c7c41ae9 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:40:37.780 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:40:37.781 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:40:37.781 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:40:37.781 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:40:37.781 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-06-01T02:40:37.781 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:40:37.781 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:40:38.089 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:40:38.089 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:40:38.089 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:40:38.089 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [], 2024-06-01T02:40:38.089 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "0/23 daemons upgraded", 2024-06-01T02:40:38.089 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 image on host smithi027" 2024-06-01T02:40:38.089 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:40:38.459 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:40:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:38 smithi003 ceph-mon[25122]: pgmap v744: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:38 smithi003 ceph-mon[25122]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:38 smithi003 ceph-mon[25122]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:38 smithi003 ceph-mon[25122]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:38 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/462258135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:40:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:38 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1266770900' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:40:39.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:38 smithi027 ceph-mon[27271]: pgmap v744: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:39.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:38 smithi027 ceph-mon[27271]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:39.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:38 smithi027 ceph-mon[27271]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:39.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:38 smithi027 ceph-mon[27271]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:39.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:38 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/462258135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:40:39.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:38 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1266770900' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:40:40.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:39 smithi003 ceph-mon[25122]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:40.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:39 smithi027 ceph-mon[27271]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:40:41.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:40 smithi003 ceph-mon[25122]: pgmap v745: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:41.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:40 smithi027 ceph-mon[27271]: pgmap v745: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:43.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:42 smithi003 ceph-mon[25122]: pgmap v746: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:43.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:42 smithi027 ceph-mon[27271]: pgmap v746: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:44 smithi027 ceph-mon[27271]: pgmap v747: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:45.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:44 smithi003 ceph-mon[25122]: pgmap v747: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:46.764 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:46 smithi027 ceph-mon[27271]: pgmap v748: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:46.765 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:46 smithi027 ceph-mon[27271]: Upgrade: Updating mgr.smithi027.tujskj 2024-06-01T02:40:46.765 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:46.765 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:40:46.765 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:40:46.765 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:46 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:40:46.765 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:46 smithi027 ceph-mon[27271]: Deploying daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:40:46.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:46 smithi003 ceph-mon[25122]: pgmap v748: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:46.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:46 smithi003 ceph-mon[25122]: Upgrade: Updating mgr.smithi027.tujskj 2024-06-01T02:40:46.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:46.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:40:46.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:40:46.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:46 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:40:46.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:46 smithi003 ceph-mon[25122]: Deploying daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:40:48.698 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:48 smithi027 ceph-mon[27271]: pgmap v749: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:48.829 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:48 smithi003 ceph-mon[25122]: pgmap v749: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:49.829 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:49 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:49.830 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:49 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:40:49.830 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:49 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:40:49.830 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:49 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:40:49.830 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:49 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:49.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:49 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:49.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:49 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:40:49.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:49 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:40:49.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:49 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:40:49.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:49 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:50.817 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:50 smithi027 ceph-mon[27271]: pgmap v750: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:50.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:50 smithi003 ceph-mon[25122]: pgmap v750: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:52.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:52 smithi003 ceph-mon[25122]: pgmap v751: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:52.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:52.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:52 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:52.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:52 smithi027 ceph-mon[27271]: pgmap v751: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:40:52.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:52.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:52 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:54.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:54 smithi003 ceph-mon[25122]: pgmap v752: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:54.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:54 smithi027 ceph-mon[27271]: pgmap v752: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:56.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:56 smithi003 ceph-mon[25122]: pgmap v753: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:56.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:56 smithi003 ceph-mon[25122]: Standby manager daemon smithi027.tujskj restarted 2024-06-01T02:40:56.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:56 smithi003 ceph-mon[25122]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:40:56.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:56 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:40:56.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:56 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:40:56.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:56 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:40:56.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:56 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:40:56.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:56 smithi027 ceph-mon[27271]: pgmap v753: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:56.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:56 smithi027 ceph-mon[27271]: Standby manager daemon smithi027.tujskj restarted 2024-06-01T02:40:56.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:56 smithi027 ceph-mon[27271]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:40:56.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:56 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:40:56.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:56 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:40:56.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:56 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:40:56.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:56 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:40:57.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:57 smithi003 ceph-mon[25122]: mgrmap e22: smithi003.njsqxz(active, since 22m), standbys: smithi027.tujskj 2024-06-01T02:40:57.848 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:57 smithi027 ceph-mon[27271]: mgrmap e22: smithi003.njsqxz(active, since 22m), standbys: smithi027.tujskj 2024-06-01T02:40:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:58 smithi003 ceph-mon[25122]: pgmap v754: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:40:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:58 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr fail", "who": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:40:58.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:58 smithi003 ceph-mon[25122]: osdmap e84: 8 total, 8 up, 8 in 2024-06-01T02:40:58.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:58 smithi027 ceph-mon[27271]: pgmap v754: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:40:58.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:58.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:58.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' 2024-06-01T02:40:58.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:40:58.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:58 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr fail", "who": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:40:58.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:58 smithi027 ceph-mon[27271]: osdmap e84: 8 total, 8 up, 8 in 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: Upgrade: Need to upgrade myself (mgr.smithi003.njsqxz) 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: Failing over to other MGR 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "mgr fail", "who": "smithi003.njsqxz"}]': finished 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: mgrmap e23: smithi027.tujskj(active, starting, since 0.998578s) 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr metadata", "who": "smithi027.tujskj", "id": "smithi027.tujskj"}]: dispatch 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:40:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: Manager daemon smithi027.tujskj is now available 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:40:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi027.tujskj/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:40:59.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi027.tujskj/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:40:59.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi027.tujskj/trash_purge_schedule"}]: dispatch 2024-06-01T02:40:59.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi027.tujskj/trash_purge_schedule"}]: dispatch 2024-06-01T02:40:59.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: Deploying cephadm binary to smithi027 2024-06-01T02:40:59.839 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:40:59 smithi003 ceph-mon[25122]: Deploying cephadm binary to smithi003 2024-06-01T02:40:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: Upgrade: Need to upgrade myself (mgr.smithi003.njsqxz) 2024-06-01T02:40:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: Failing over to other MGR 2024-06-01T02:40:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.14204 172.21.15.3:0/194661024' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "mgr fail", "who": "smithi003.njsqxz"}]': finished 2024-06-01T02:40:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: mgrmap e23: smithi027.tujskj(active, starting, since 0.998578s) 2024-06-01T02:40:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:40:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:40:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:40:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:40:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr metadata", "who": "smithi027.tujskj", "id": "smithi027.tujskj"}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: Manager daemon smithi027.tujskj is now available 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:40:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:40:59.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:40:59.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi027.tujskj/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:40:59.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi027.tujskj/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:40:59.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi027.tujskj/trash_purge_schedule"}]: dispatch 2024-06-01T02:40:59.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi027.tujskj/trash_purge_schedule"}]: dispatch 2024-06-01T02:40:59.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: Deploying cephadm binary to smithi027 2024-06-01T02:40:59.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:40:59 smithi027 ceph-mon[27271]: Deploying cephadm binary to smithi003 2024-06-01T02:41:01.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:00 smithi003 ceph-mon[25122]: mgrmap e24: smithi027.tujskj(active, since 1.99936s) 2024-06-01T02:41:01.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:00 smithi003 ceph-mon[25122]: pgmap v3: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:41:01.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:00 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:01.187 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:00 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:01.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:00 smithi027 ceph-mon[27271]: mgrmap e24: smithi027.tujskj(active, since 1.99936s) 2024-06-01T02:41:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:00 smithi027 ceph-mon[27271]: pgmap v3: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:41:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:00 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:00 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:02.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:01 smithi003 ceph-mon[25122]: pgmap v4: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:41:02.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:01 smithi003 ceph-mon[25122]: [01/Jun/2024:02:41:01] ENGINE Bus STARTING 2024-06-01T02:41:02.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:01 smithi003 ceph-mon[25122]: [01/Jun/2024:02:41:01] ENGINE Serving on https://172.21.15.27:7150 2024-06-01T02:41:02.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:01 smithi027 ceph-mon[27271]: pgmap v4: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:41:02.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:01 smithi027 ceph-mon[27271]: [01/Jun/2024:02:41:01] ENGINE Bus STARTING 2024-06-01T02:41:02.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:01 smithi027 ceph-mon[27271]: [01/Jun/2024:02:41:01] ENGINE Serving on https://172.21.15.27:7150 2024-06-01T02:41:03.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:02 smithi003 ceph-mon[25122]: [01/Jun/2024:02:41:01] ENGINE Serving on http://172.21.15.27:8765 2024-06-01T02:41:03.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:02 smithi003 ceph-mon[25122]: [01/Jun/2024:02:41:01] ENGINE Bus STARTED 2024-06-01T02:41:03.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:02 smithi003 ceph-mon[25122]: mgrmap e25: smithi027.tujskj(active, since 4s) 2024-06-01T02:41:03.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:02 smithi027 ceph-mon[27271]: [01/Jun/2024:02:41:01] ENGINE Serving on http://172.21.15.27:8765 2024-06-01T02:41:03.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:02 smithi027 ceph-mon[27271]: [01/Jun/2024:02:41:01] ENGINE Bus STARTED 2024-06-01T02:41:03.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:02 smithi027 ceph-mon[27271]: mgrmap e25: smithi027.tujskj(active, since 4s) 2024-06-01T02:41:04.254 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:03 smithi003 ceph-mon[25122]: pgmap v5: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:41:04.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:03 smithi027 ceph-mon[27271]: pgmap v5: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:41:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:06 smithi027 ceph-mon[27271]: pgmap v6: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:41:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:06 smithi027 ceph-mon[27271]: Standby manager daemon smithi003.njsqxz started 2024-06-01T02:41:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:06 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.3:0/3321920868' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi003.njsqxz/crt"}]: dispatch 2024-06-01T02:41:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:06 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.3:0/3321920868' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:41:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:06 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.3:0/3321920868' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi003.njsqxz/key"}]: dispatch 2024-06-01T02:41:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:06 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.3:0/3321920868' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:41:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:06 smithi003 ceph-mon[25122]: pgmap v6: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:41:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:06 smithi003 ceph-mon[25122]: Standby manager daemon smithi003.njsqxz started 2024-06-01T02:41:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:06 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.3:0/3321920868' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi003.njsqxz/crt"}]: dispatch 2024-06-01T02:41:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:06 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.3:0/3321920868' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:41:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:06 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.3:0/3321920868' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi003.njsqxz/key"}]: dispatch 2024-06-01T02:41:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:06 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.3:0/3321920868' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:41:07.111 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:07 smithi027 ceph-mon[27271]: mgrmap e26: smithi027.tujskj(active, since 8s), standbys: smithi003.njsqxz 2024-06-01T02:41:07.112 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:07 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:41:07.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:07 smithi003 ceph-mon[25122]: mgrmap e26: smithi027.tujskj(active, since 8s), standbys: smithi003.njsqxz 2024-06-01T02:41:07.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:07 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:41:08.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:08 smithi003 ceph-mon[25122]: pgmap v7: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-01T02:41:08.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:08 smithi027 ceph-mon[27271]: pgmap v7: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-01T02:41:08.849 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:41:09.544 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:41:09.544 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (21m) 0s ago 23m 26.3M - 0.23.0 ba2b418f427c ceda73487cd9 2024-06-01T02:41:09.544 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (23m) 0s ago 23m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:41:09.544 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (22m) 1s ago 22m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:41:09.544 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (21m) 0s ago 22m 52.2M - 8.3.5 dad864ee21e9 32cd8820aa5e 2024-06-01T02:41:09.544 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (17m) 0s ago 17m 40.1M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:41:09.544 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (17m) 1s ago 17m 17.3M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:41:09.544 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:9283 running (24m) 0s ago 24m 382M - 17.2.0 e1d6a67b021e ac6fbe8a9d2d 2024-06-01T02:41:09.544 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283 running (21s) 1s ago 22m 492M - 19.0.0-2946-g3d89accf b045ac545f82 676453f46f87 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (24m) 0s ago 24m 63.0M 2048M 17.2.0 e1d6a67b021e 108dda5fe7cc 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (22m) 1s ago 22m 47.1M 2048M 17.2.0 e1d6a67b021e edc537c54a3a 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (17m) 0s ago 17m 249M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (16m) 1s ago 16m 87.4M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (22m) 0s ago 22m 17.9M - 1.3.1 1dbe0e931976 6338b03aebaf 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (22m) 1s ago 22m 20.1M - 1.3.1 1dbe0e931976 3ee520f9eebb 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (21m) 0s ago 21m 441M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (21m) 0s ago 21m 359M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (20m) 0s ago 20m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (20m) 0s ago 20m 369M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (20m) 1s ago 20m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (19m) 1s ago 19m 461M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (19m) 1s ago 19m 374M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (18m) 1s ago 18m 616M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:41:09.545 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (22m) 0s ago 22m 70.8M - 2.33.4 514e6a882f6e 70b2c7c41ae9 2024-06-01T02:41:09.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:09 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:09.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:09 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:09.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:09 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:09.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:09 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:09.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:09 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:09.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:09 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:09.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:09 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:09.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:09 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:09.964 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:41:09.964 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:41:09.964 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:41:09.964 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:41:09.964 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:41:09.964 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-01T02:41:09.964 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 1 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 1 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:41:09.965 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:41:10.320 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:41:10.320 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:41:10.320 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:41:10.320 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:41:10.320 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [], 2024-06-01T02:41:10.320 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "1/23 daemons upgraded", 2024-06-01T02:41:10.320 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "", 2024-06-01T02:41:10.320 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:41:10.321 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:41:10.489 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:10 smithi027 ceph-mon[27271]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:10.489 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:10 smithi027 ceph-mon[27271]: pgmap v8: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-01T02:41:10.489 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:10 smithi027 ceph-mon[27271]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:10.489 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:10 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:10.489 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:10 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:10.489 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:10 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/551099859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:41:10.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:10 smithi003 ceph-mon[25122]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:10.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:10 smithi003 ceph-mon[25122]: pgmap v8: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-01T02:41:10.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:10 smithi003 ceph-mon[25122]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:10.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:10 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:10.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:10 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:10.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:10 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/551099859' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:41:10.735 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:41:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:11 smithi003 ceph-mon[25122]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:11 smithi003 ceph-mon[25122]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:11 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:11 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:11 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/557226726' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:41:11.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:11 smithi027 ceph-mon[27271]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:11 smithi027 ceph-mon[27271]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:11 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:11 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:11 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/557226726' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:41:12.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:12 smithi003 ceph-mon[25122]: pgmap v9: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-01T02:41:12.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:12 smithi027 ceph-mon[27271]: pgmap v9: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-01T02:41:14.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:14 smithi003 ceph-mon[25122]: pgmap v10: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:41:14.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:14 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:41:14.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:14 smithi027 ceph-mon[27271]: pgmap v10: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:41:14.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:14 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:41:16.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:16 smithi003 ceph-mon[25122]: pgmap v11: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:41:16.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:16 smithi027 ceph-mon[27271]: pgmap v11: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:41:18.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:18 smithi003 ceph-mon[25122]: pgmap v12: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:41:18.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:18 smithi027 ceph-mon[27271]: pgmap v12: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:41:20.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:20 smithi003 ceph-mon[25122]: pgmap v13: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:20.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:20 smithi027 ceph-mon[27271]: pgmap v13: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:21.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:21 smithi003 ceph-mon[25122]: pgmap v14: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:41:21.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:21 smithi027 ceph-mon[27271]: pgmap v14: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:41:24.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:24.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:24.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:41:24.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:41:24.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:41:24.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-01T02:41:24.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: pgmap v15: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:24.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:41:24.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:41:24.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: Updating smithi003:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:41:24.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:23 smithi027 ceph-mon[27271]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:41:24.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:24.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:24.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:41:24.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:41:24.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:41:24.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-01T02:41:24.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: pgmap v15: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:24.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:41:24.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:41:24.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: Updating smithi003:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:41:24.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:23 smithi003 ceph-mon[25122]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:41:25.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: Reconfiguring alertmanager.smithi003 (dependencies changed)... 2024-06-01T02:41:25.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:24 smithi003 ceph-mon[25122]: Reconfiguring daemon alertmanager.smithi003 on smithi003 2024-06-01T02:41:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:41:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:41:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:25.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: Reconfiguring alertmanager.smithi003 (dependencies changed)... 2024-06-01T02:41:25.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:24 smithi027 ceph-mon[27271]: Reconfiguring daemon alertmanager.smithi003 on smithi003 2024-06-01T02:41:26.194 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:26 smithi003 ceph-mon[25122]: pgmap v16: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:26.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:26 smithi027 ceph-mon[27271]: pgmap v16: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:28.307 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:28 smithi003 ceph-mon[25122]: pgmap v17: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:28.351 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:28 smithi027 ceph-mon[27271]: pgmap v17: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:29.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:29 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:41:29.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:29 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:41:30.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: pgmap v18: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: Reconfiguring grafana.smithi003 (dependencies changed)... 2024-06-01T02:41:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-01T02:41:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-01T02:41:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-01T02:41:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:30 smithi003 ceph-mon[25122]: Reconfiguring daemon grafana.smithi003 on smithi003 2024-06-01T02:41:30.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: pgmap v18: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:30.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:30.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:30.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: Reconfiguring grafana.smithi003 (dependencies changed)... 2024-06-01T02:41:30.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-01T02:41:30.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:30.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:30.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-01T02:41:30.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-01T02:41:30.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:30 smithi027 ceph-mon[27271]: Reconfiguring daemon grafana.smithi003 on smithi003 2024-06-01T02:41:32.325 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:32 smithi003 ceph-mon[25122]: pgmap v19: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:41:32.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:32 smithi027 ceph-mon[27271]: pgmap v19: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:41:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:34 smithi003 ceph-mon[25122]: pgmap v20: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:34 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:34 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:34.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:34 smithi027 ceph-mon[27271]: pgmap v20: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:34.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:34 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:34.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:34 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:35.440 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:35 smithi003 ceph-mon[25122]: Reconfiguring node-exporter.smithi003 (dependencies changed)... 2024-06-01T02:41:35.440 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:35 smithi003 ceph-mon[25122]: Reconfiguring daemon node-exporter.smithi003 on smithi003 2024-06-01T02:41:35.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:35 smithi027 ceph-mon[27271]: Reconfiguring node-exporter.smithi003 (dependencies changed)... 2024-06-01T02:41:35.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:35 smithi027 ceph-mon[27271]: Reconfiguring daemon node-exporter.smithi003 on smithi003 2024-06-01T02:41:36.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:36 smithi003 ceph-mon[25122]: pgmap v21: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:36.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:36 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:36.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:36 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:36 smithi027 ceph-mon[27271]: pgmap v21: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:36 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:36 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:37.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:37 smithi003 ceph-mon[25122]: Reconfiguring prometheus.smithi003 (dependencies changed)... 2024-06-01T02:41:37.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:37 smithi003 ceph-mon[25122]: Reconfiguring daemon prometheus.smithi003 on smithi003 2024-06-01T02:41:37.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:37 smithi027 ceph-mon[27271]: Reconfiguring prometheus.smithi003 (dependencies changed)... 2024-06-01T02:41:37.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:37 smithi027 ceph-mon[27271]: Reconfiguring daemon prometheus.smithi003 on smithi003 2024-06-01T02:41:38.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:38 smithi003 ceph-mon[25122]: pgmap v22: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:38.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:38 smithi027 ceph-mon[27271]: pgmap v22: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:40.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:40 smithi003 ceph-mon[25122]: pgmap v23: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:40.601 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:40 smithi027 ceph-mon[27271]: pgmap v23: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:41.110 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:41:41.755 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:41 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:41.756 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:41 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:41.756 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:41 smithi027 ceph-mon[27271]: Reconfiguring node-exporter.smithi027 (dependencies changed)... 2024-06-01T02:41:41.756 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:41 smithi027 ceph-mon[27271]: Reconfiguring daemon node-exporter.smithi027 on smithi027 2024-06-01T02:41:41.756 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:41 smithi027 ceph-mon[27271]: pgmap v24: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:41:41.756 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:41 smithi027 ceph-mon[27271]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 starting - - - - 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (23m) 32s ago 23m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (23m) 33s ago 23m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 starting - - - - 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (17m) 32s ago 17m 40.1M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (17m) 33s ago 17m 17.3M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:9283 running (24m) 32s ago 24m 382M - 17.2.0 e1d6a67b021e ac6fbe8a9d2d 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283 running (53s) 33s ago 23m 492M - 19.0.0-2946-g3d89accf b045ac545f82 676453f46f87 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (24m) 32s ago 25m 63.0M 2048M 17.2.0 e1d6a67b021e 108dda5fe7cc 2024-06-01T02:41:41.792 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (23m) 33s ago 23m 47.1M 2048M 17.2.0 e1d6a67b021e edc537c54a3a 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (17m) 32s ago 17m 249M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (17m) 33s ago 17m 87.4M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 starting - - - - 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (22m) 33s ago 22m 20.1M - 1.3.1 1dbe0e931976 3ee520f9eebb 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (22m) 32s ago 22m 441M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (21m) 32s ago 21m 359M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (21m) 32s ago 21m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (20m) 32s ago 20m 369M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (20m) 33s ago 20m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (20m) 33s ago 20m 461M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (19m) 33s ago 19m 374M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:41:41.793 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (19m) 33s ago 19m 616M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:41:41.794 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 starting - - - - 2024-06-01T02:41:41.794 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:41 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:41.794 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:41 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:41.794 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:41 smithi003 ceph-mon[25122]: Reconfiguring node-exporter.smithi027 (dependencies changed)... 2024-06-01T02:41:41.794 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:41 smithi003 ceph-mon[25122]: Reconfiguring daemon node-exporter.smithi027 on smithi027 2024-06-01T02:41:41.794 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:41 smithi003 ceph-mon[25122]: pgmap v24: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:41:41.794 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:41 smithi003 ceph-mon[25122]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:42.207 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 1 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:41:42.208 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:41:42.209 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:41:42.209 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:41:42.209 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:41:42.209 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:41:42.209 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-01T02:41:42.209 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 1 2024-06-01T02:41:42.209 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:41:42.209 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1856226260' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi003.front.sepia.ceph.com:9093"}]: dispatch 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi003.front.sepia.ceph.com:9093"}]: dispatch 2024-06-01T02:41:42.527 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.528 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:41:42.528 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:41:42.528 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi003.front.sepia.ceph.com:3000"}]: dispatch 2024-06-01T02:41:42.528 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi003.front.sepia.ceph.com:3000"}]: dispatch 2024-06-01T02:41:42.528 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.528 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:41:42.528 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi003.front.sepia.ceph.com:9095"}]: dispatch 2024-06-01T02:41:42.528 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.528 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:42 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:41:42.559 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:41:42.560 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:41:42.560 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:41:42.560 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:41:42.560 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [], 2024-06-01T02:41:42.560 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "1/23 daemons upgraded", 2024-06-01T02:41:42.560 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading mgr daemons", 2024-06-01T02:41:42.560 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:41:42.560 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:41:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1856226260' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:41:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:41:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:41:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi003.front.sepia.ceph.com:9093"}]: dispatch 2024-06-01T02:41:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi003.front.sepia.ceph.com:9093"}]: dispatch 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi003.front.sepia.ceph.com:3000"}]: dispatch 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi003.front.sepia.ceph.com:3000"}]: dispatch 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi003.front.sepia.ceph.com:9095"}]: dispatch 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:42.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:42 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:41:43.007 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:41:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:43 smithi003 ceph-mon[25122]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:41:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:43 smithi003 ceph-mon[25122]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi003.front.sepia.ceph.com:9095"}]: dispatch 2024-06-01T02:41:43.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:43 smithi003 ceph-mon[25122]: Upgrade: Need to upgrade myself (mgr.smithi027.tujskj) 2024-06-01T02:41:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:43 smithi003 ceph-mon[25122]: Upgrade: Need to upgrade myself (mgr.smithi027.tujskj) 2024-06-01T02:41:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:43 smithi003 ceph-mon[25122]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:43 smithi003 ceph-mon[25122]: pgmap v25: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:43 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/691859312' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:41:43.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:43 smithi027 ceph-mon[27271]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:41:43.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:43 smithi027 ceph-mon[27271]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi003.front.sepia.ceph.com:9095"}]: dispatch 2024-06-01T02:41:43.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:43 smithi027 ceph-mon[27271]: Upgrade: Need to upgrade myself (mgr.smithi027.tujskj) 2024-06-01T02:41:43.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:43 smithi027 ceph-mon[27271]: Upgrade: Need to upgrade myself (mgr.smithi027.tujskj) 2024-06-01T02:41:43.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:43 smithi027 ceph-mon[27271]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:41:43.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:43 smithi027 ceph-mon[27271]: pgmap v25: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:43.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:43 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/691859312' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:41:44.824 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:44 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:41:44.824 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:44 smithi003 ceph-mon[25122]: Upgrade: Updating mgr.smithi003.njsqxz 2024-06-01T02:41:44.824 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:44 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:44.824 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:44 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi003.njsqxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:41:44.824 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:44 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi003.njsqxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:41:44.824 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:44 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:41:44.825 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:44 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:41:44.825 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:44 smithi003 ceph-mon[25122]: Deploying daemon mgr.smithi003.njsqxz on smithi003 2024-06-01T02:41:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:44 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:41:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:44 smithi027 ceph-mon[27271]: Upgrade: Updating mgr.smithi003.njsqxz 2024-06-01T02:41:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:44 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:44 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi003.njsqxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:41:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:44 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi003.njsqxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:41:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:44 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:41:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:44 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:41:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:44 smithi027 ceph-mon[27271]: Deploying daemon mgr.smithi003.njsqxz on smithi003 2024-06-01T02:41:45.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:45 smithi003 ceph-mon[25122]: pgmap v26: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:45.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:45 smithi027 ceph-mon[27271]: pgmap v26: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:48.025 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:47 smithi003 ceph-mon[25122]: pgmap v27: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:48.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:47 smithi027 ceph-mon[27271]: pgmap v27: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:49.217 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:49 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:49.217 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:49 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:49.217 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:49 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:41:49.261 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:49 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:49.261 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:49 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:49.261 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:49 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:41:50.450 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:50 smithi003 ceph-mon[25122]: pgmap v28: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:50.475 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:50 smithi027 ceph-mon[27271]: pgmap v28: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:52.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:52 smithi003 ceph-mon[25122]: pgmap v29: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:41:52.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:52 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:52.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:52 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:52 smithi027 ceph-mon[27271]: pgmap v29: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:41:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:52 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:52 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:54.585 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:54 smithi003 ceph-mon[25122]: pgmap v30: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:54.585 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:54 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:54 smithi027 ceph-mon[27271]: pgmap v30: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:54 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:55.492 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:55 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:55 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:56.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:56 smithi003 ceph-mon[25122]: pgmap v31: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:56.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:56 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:56.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:56 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:56 smithi003 ceph-mon[25122]: Standby manager daemon smithi003.njsqxz restarted 2024-06-01T02:41:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:56 smithi003 ceph-mon[25122]: Standby manager daemon smithi003.njsqxz started 2024-06-01T02:41:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:56 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi003.njsqxz/crt"}]: dispatch 2024-06-01T02:41:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:56 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:41:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:56 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi003.njsqxz/key"}]: dispatch 2024-06-01T02:41:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:56 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:41:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:56 smithi027 ceph-mon[27271]: pgmap v31: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:56 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:56 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:56 smithi027 ceph-mon[27271]: Standby manager daemon smithi003.njsqxz restarted 2024-06-01T02:41:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:56 smithi027 ceph-mon[27271]: Standby manager daemon smithi003.njsqxz started 2024-06-01T02:41:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:56 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi003.njsqxz/crt"}]: dispatch 2024-06-01T02:41:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:56 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:41:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:56 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi003.njsqxz/key"}]: dispatch 2024-06-01T02:41:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:56 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:41:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:57 smithi027 ceph-mon[27271]: mgrmap e27: smithi027.tujskj(active, since 58s), standbys: smithi003.njsqxz 2024-06-01T02:41:57.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:57 smithi003 ceph-mon[25122]: mgrmap e27: smithi027.tujskj(active, since 58s), standbys: smithi003.njsqxz 2024-06-01T02:41:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:58 smithi027 ceph-mon[27271]: pgmap v32: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:58.660 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:58 smithi003 ceph-mon[25122]: pgmap v32: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:41:59.965 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:59.965 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:59.965 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:41:59.965 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:41:59.965 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:59.965 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:59.965 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:41:59.965 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: Upgrade: Need to upgrade myself (mgr.smithi027.tujskj) 2024-06-01T02:41:59.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: Upgrade: Need to upgrade myself (mgr.smithi027.tujskj) 2024-06-01T02:41:59.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: Failing over to other MGR 2024-06-01T02:41:59.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr fail", "who": "smithi027.tujskj"}]: dispatch 2024-06-01T02:41:59.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr fail", "who": "smithi027.tujskj"}]: dispatch 2024-06-01T02:41:59.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: osdmap e85: 8 total, 8 up, 8 in 2024-06-01T02:41:59.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: pgmap v34: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:41:59.966 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:41:59 smithi003 ceph-mon[25122]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:41:59.966 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: Upgrade: Need to upgrade myself (mgr.smithi027.tujskj) 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: Upgrade: Need to upgrade myself (mgr.smithi027.tujskj) 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: Failing over to other MGR 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr fail", "who": "smithi027.tujskj"}]: dispatch 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "mgr fail", "who": "smithi027.tujskj"}]: dispatch 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: osdmap e85: 8 total, 8 up, 8 in 2024-06-01T02:41:59.967 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: pgmap v34: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:41:59.968 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:41:59 smithi027 ceph-mon[27271]: from='mgr.24449 172.21.15.27:0/4168754473' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:42:00.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd='[{"prefix": "mgr fail", "who": "smithi027.tujskj"}]': finished 2024-06-01T02:42:00.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: mgrmap e28: smithi003.njsqxz(active, starting, since 0.864583s) 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:42:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:42:00.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:42:00.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:42:00.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:42:00.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: Manager daemon smithi003.njsqxz is now available 2024-06-01T02:42:00.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:00.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:42:00.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:42:00.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:00 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:42:01.068 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.24449 ' entity='mgr.smithi027.tujskj' cmd='[{"prefix": "mgr fail", "who": "smithi027.tujskj"}]': finished 2024-06-01T02:42:01.068 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: mgrmap e28: smithi003.njsqxz(active, starting, since 0.864583s) 2024-06-01T02:42:01.068 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:42:01.068 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:42:01.068 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:42:01.068 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:42:01.068 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: Manager daemon smithi003.njsqxz is now available 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:42:01.069 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:42:01.070 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:00 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:42:02.050 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:01 smithi027 ceph-mon[27271]: [01/Jun/2024:02:42:00] ENGINE Bus STARTING 2024-06-01T02:42:02.051 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:01 smithi027 ceph-mon[27271]: [01/Jun/2024:02:42:00] ENGINE Serving on https://172.21.15.3:7150 2024-06-01T02:42:02.051 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:01 smithi027 ceph-mon[27271]: mgrmap e29: smithi003.njsqxz(active, since 1.8719s) 2024-06-01T02:42:02.051 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:01 smithi027 ceph-mon[27271]: pgmap v3: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:02.051 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:01 smithi027 ceph-mon[27271]: [01/Jun/2024:02:42:00] ENGINE Serving on http://172.21.15.3:8765 2024-06-01T02:42:02.051 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:01 smithi027 ceph-mon[27271]: [01/Jun/2024:02:42:00] ENGINE Bus STARTED 2024-06-01T02:42:02.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:01 smithi003 ceph-mon[25122]: [01/Jun/2024:02:42:00] ENGINE Bus STARTING 2024-06-01T02:42:02.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:01 smithi003 ceph-mon[25122]: [01/Jun/2024:02:42:00] ENGINE Serving on https://172.21.15.3:7150 2024-06-01T02:42:02.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:01 smithi003 ceph-mon[25122]: mgrmap e29: smithi003.njsqxz(active, since 1.8719s) 2024-06-01T02:42:02.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:01 smithi003 ceph-mon[25122]: pgmap v3: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:02.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:01 smithi003 ceph-mon[25122]: [01/Jun/2024:02:42:00] ENGINE Serving on http://172.21.15.3:8765 2024-06-01T02:42:02.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:01 smithi003 ceph-mon[25122]: [01/Jun/2024:02:42:00] ENGINE Bus STARTED 2024-06-01T02:42:03.053 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:02 smithi003 ceph-mon[25122]: pgmap v4: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:03.053 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:02 smithi003 ceph-mon[25122]: mgrmap e30: smithi003.njsqxz(active, since 2s) 2024-06-01T02:42:03.055 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:02 smithi027 ceph-mon[27271]: pgmap v4: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:03.055 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:02 smithi027 ceph-mon[27271]: mgrmap e30: smithi003.njsqxz(active, since 2s) 2024-06-01T02:42:04.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:04 smithi003 ceph-mon[25122]: pgmap v5: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:04.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:04 smithi003 ceph-mon[25122]: mgrmap e31: smithi003.njsqxz(active, since 4s) 2024-06-01T02:42:04.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:04 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:04.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:04 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:05.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:04 smithi027 ceph-mon[27271]: pgmap v5: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:05.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:04 smithi027 ceph-mon[27271]: mgrmap e31: smithi003.njsqxz(active, since 4s) 2024-06-01T02:42:05.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:04 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:05.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:04 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:06.338 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:06 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:06.338 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:06 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:06.338 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:06 smithi027 ceph-mon[27271]: pgmap v6: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:06.338 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:06 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:06.339 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:06 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:06.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:06 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:06.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:06 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:06.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:06 smithi003 ceph-mon[25122]: pgmap v6: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:06.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:06 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:06.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:06 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:08.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:08 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:08.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:08 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:08.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:08 smithi003 ceph-mon[25122]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:42:08.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:08 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/3951030804' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:42:08.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:08 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/3951030804' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:42:08.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:08 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/3951030804' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:42:08.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:08 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/3951030804' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:42:08.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:08 smithi003 ceph-mon[25122]: pgmap v7: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-01T02:42:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:08 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:08 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:08 smithi027 ceph-mon[27271]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:42:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:08 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/3951030804' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:42:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:08 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/3951030804' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:42:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:08 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/3951030804' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:42:08.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:08 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/3951030804' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:42:08.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:08 smithi027 ceph-mon[27271]: pgmap v7: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-01T02:42:09.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:09 smithi003 ceph-mon[25122]: mgrmap e32: smithi003.njsqxz(active, since 9s), standbys: smithi027.tujskj 2024-06-01T02:42:09.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:09 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi027.tujskj", "id": "smithi027.tujskj"}]: dispatch 2024-06-01T02:42:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:09 smithi027 ceph-mon[27271]: mgrmap e32: smithi003.njsqxz(active, since 9s), standbys: smithi027.tujskj 2024-06-01T02:42:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:09 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi027.tujskj", "id": "smithi027.tujskj"}]: dispatch 2024-06-01T02:42:10.397 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:10 smithi003 ceph-mon[25122]: pgmap v8: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-01T02:42:10.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:10 smithi027 ceph-mon[27271]: pgmap v8: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-01T02:42:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:42:11.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:42:11.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-01T02:42:11.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:42:11.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:42:11.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: Updating smithi003:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:42:11.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:11 smithi003 ceph-mon[25122]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: Updating smithi003:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:42:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:11 smithi027 ceph-mon[27271]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:42:12.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:42:12.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:42:12.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: Reconfiguring prometheus.smithi003 (dependencies changed)... 2024-06-01T02:42:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: pgmap v9: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-01T02:42:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:12 smithi003 ceph-mon[25122]: Reconfiguring daemon prometheus.smithi003 on smithi003 2024-06-01T02:42:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:42:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:42:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:12.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: Reconfiguring prometheus.smithi003 (dependencies changed)... 2024-06-01T02:42:12.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: pgmap v9: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-01T02:42:12.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:12 smithi027 ceph-mon[27271]: Reconfiguring daemon prometheus.smithi003 on smithi003 2024-06-01T02:42:13.372 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:42:14.068 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (45s) 8s ago 24m 13.5M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (24m) 8s ago 24m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (23m) 10s ago 23m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (40s) 8s ago 23m 44.5M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (18m) 8s ago 18m 40.1M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (18m) 10s ago 18m 17.6M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (26s) 8s ago 25m 475M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283 running (85s) 10s ago 23m 73.1M - 19.0.0-2946-g3d89accf b045ac545f82 676453f46f87 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (25m) 8s ago 25m 67.5M 2048M 17.2.0 e1d6a67b021e 108dda5fe7cc 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (23m) 10s ago 23m 50.5M 2048M 17.2.0 e1d6a67b021e edc537c54a3a 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (18m) 8s ago 18m 265M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:42:14.069 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (18m) 10s ago 18m 87.4M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (38s) 8s ago 23m 8656k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (32s) 10s ago 23m 8744k - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (22m) 8s ago 22m 442M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (22m) 8s ago 22m 360M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (21m) 8s ago 21m 491M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (21m) 8s ago 21m 370M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (21m) 10s ago 21m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (20m) 10s ago 20m 459M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (20m) 10s ago 20m 374M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (19m) 10s ago 19m 617M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:42:14.070 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (34s) 8s ago 23m 53.4M - 2.33.4 514e6a882f6e b7d7911991a0 2024-06-01T02:42:14.494 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:42:14.494 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:42:14.494 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:42:14.494 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:42:14.494 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:42:14.494 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:42:14.494 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:42:14.494 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:42:14.494 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:42:14.495 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:42:14.495 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:42:14.495 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:42:14.495 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:42:14.495 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:42:14.495 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-06-01T02:42:14.495 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:42:14.495 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:42:14.495 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "2/23 daemons upgraded", 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "", 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:42:14.845 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:42:15.061 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:14 smithi003 ceph-mon[25122]: from='client.14830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:15.061 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:14 smithi003 ceph-mon[25122]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:15.061 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:14 smithi003 ceph-mon[25122]: pgmap v10: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:42:15.062 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:14 smithi003 ceph-mon[25122]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:15.062 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:14 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/1633507069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:42:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:14 smithi027 ceph-mon[27271]: from='client.14830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:14 smithi027 ceph-mon[27271]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:14 smithi027 ceph-mon[27271]: pgmap v10: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:42:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:14 smithi027 ceph-mon[27271]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:14 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1633507069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:42:15.257 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:42:15.987 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:15 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:42:15.987 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:15 smithi003 ceph-mon[25122]: from='client.14846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:15.987 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:15 smithi003 ceph-mon[25122]: from='client.? 172.21.15.3:0/330998486' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:42:16.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:15 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:42:16.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:15 smithi027 ceph-mon[27271]: from='client.14846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:16.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:15 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/330998486' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:42:17.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:16 smithi003 ceph-mon[25122]: pgmap v11: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:42:17.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:16 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:17.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:16 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:17.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:16 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:42:17.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:16 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:16 smithi027 ceph-mon[27271]: pgmap v11: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:42:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:16 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:16 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:16 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:42:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:16 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:18.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:17 smithi003 ceph-mon[25122]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:42:18.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:17 smithi027 ceph-mon[27271]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:42:18.817 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:18 smithi027 ceph-mon[27271]: pgmap v12: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:42:18.818 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:18 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:42:19.067 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:18 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:42:19.067 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:18 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:19.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:18 smithi003 ceph-mon[25122]: pgmap v12: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:42:19.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:18 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:42:19.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:18 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:42:19.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:18 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:20.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:19 smithi003 ceph-mon[25122]: Upgrade: Updating mgr.smithi027.tujskj 2024-06-01T02:42:20.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:19 smithi003 ceph-mon[25122]: Deploying daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:42:20.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:19 smithi027 ceph-mon[27271]: Upgrade: Updating mgr.smithi027.tujskj 2024-06-01T02:42:20.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:19 smithi027 ceph-mon[27271]: Deploying daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:42:21.018 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:20 smithi027 ceph-mon[27271]: pgmap v13: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:21.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:20 smithi003 ceph-mon[25122]: pgmap v13: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:23.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:22 smithi003 ceph-mon[25122]: pgmap v14: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:42:23.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:22 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:23.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:22 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:23.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:22 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:23.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:22 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:23.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:22 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:23.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:22 smithi027 ceph-mon[27271]: pgmap v14: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:42:23.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:22 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:23.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:22 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:23.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:22 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:23.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:22 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:23.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:22 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:25.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:24 smithi003 ceph-mon[25122]: pgmap v15: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:25.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:24 smithi027 ceph-mon[27271]: pgmap v15: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:27.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:26 smithi003 ceph-mon[25122]: pgmap v16: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:27.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:26 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:27.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:26 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:27.101 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:26 smithi027 ceph-mon[27271]: pgmap v16: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:27.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:26 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:27.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:26 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:29.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:28 smithi003 ceph-mon[25122]: pgmap v17: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:29.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:28 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:29.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:28 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:29.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:28 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:29.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:28 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:29.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:28 smithi027 ceph-mon[27271]: pgmap v17: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:29.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:28 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:29.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:28 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:29.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:28 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:29.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:28 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:30.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:30 smithi027 ceph-mon[27271]: Standby manager daemon smithi027.tujskj restarted 2024-06-01T02:42:30.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:30 smithi027 ceph-mon[27271]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:42:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:30 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/1129858567' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:42:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:30 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/1129858567' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:42:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:30 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/1129858567' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:42:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:30 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/1129858567' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:42:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:30 smithi027 ceph-mon[27271]: pgmap v18: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:30 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:42:30.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:30 smithi003 ceph-mon[25122]: Standby manager daemon smithi027.tujskj restarted 2024-06-01T02:42:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:30 smithi003 ceph-mon[25122]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:42:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:30 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/1129858567' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:42:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:30 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/1129858567' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:42:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:30 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/1129858567' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:42:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:30 smithi003 ceph-mon[25122]: from='mgr.? 172.21.15.27:0/1129858567' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:42:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:30 smithi003 ceph-mon[25122]: pgmap v18: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:30.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:30 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:42:31.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:31 smithi027 ceph-mon[27271]: mgrmap e33: smithi003.njsqxz(active, since 31s), standbys: smithi027.tujskj 2024-06-01T02:42:31.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:31 smithi003 ceph-mon[25122]: mgrmap e33: smithi003.njsqxz(active, since 31s), standbys: smithi027.tujskj 2024-06-01T02:42:32.515 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: Upgrade: Setting container_image for all mgr 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi003.njsqxz"}]: dispatch 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi003.njsqxz"}]': finished 2024-06-01T02:42:32.516 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi027.tujskj"}]: dispatch 2024-06-01T02:42:32.517 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi027.tujskj"}]': finished 2024-06-01T02:42:32.517 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-01T02:42:32.517 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:32 smithi003 ceph-mon[25122]: pgmap v19: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:42:32.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: Upgrade: Setting container_image for all mgr 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi003.njsqxz"}]: dispatch 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi003.njsqxz"}]': finished 2024-06-01T02:42:32.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi027.tujskj"}]: dispatch 2024-06-01T02:42:32.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi027.tujskj"}]': finished 2024-06-01T02:42:32.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-01T02:42:32.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:32 smithi027 ceph-mon[27271]: pgmap v19: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:42:34.377 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:34 smithi003 ceph-mon[25122]: Upgrade: Updating mon.smithi003 2024-06-01T02:42:34.377 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:34 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:34.377 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:34 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:42:34.377 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:34 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:42:34.377 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:34 smithi003 ceph-mon[25122]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:34.377 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:34 smithi003 ceph-mon[25122]: Deploying daemon mon.smithi003 on smithi003 2024-06-01T02:42:34.377 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:34 smithi003 ceph-mon[25122]: pgmap v20: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:34.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:34 smithi027 ceph-mon[27271]: Upgrade: Updating mon.smithi003 2024-06-01T02:42:34.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:34 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:34.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:34 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:42:34.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:34 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:42:34.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:34 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:34.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:34 smithi027 ceph-mon[27271]: Deploying daemon mon.smithi003 on smithi003 2024-06-01T02:42:34.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:34 smithi027 ceph-mon[27271]: pgmap v20: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:35.375 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:35 smithi003 systemd[1]: Stopping Ceph mon.smithi003 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:42:35.758 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:35 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003[25118]: 2024-06-01T02:42:35.375+0000 7fd5da825700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi003 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:42:35.758 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:35 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003[25118]: 2024-06-01T02:42:35.375+0000 7fd5da825700 -1 mon.smithi003@0(leader) e2 *** Got Signal Terminated *** 2024-06-01T02:42:35.759 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:35 smithi003 podman[67192]: 2024-06-01 02:42:35.487845492 +0000 UTC m=+0.209940685 container died 108dda5fe7cc0c23a6888f3c445309bfa86e8a0b0a51ab966affb69af2b19757 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, maintainer=Guillaume Abrioux , distribution-scope=public, io.openshift.expose-services=, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_BRANCH=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.buildah.version=1.19.8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-06-01T02:42:36.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:35 smithi003 podman[67192]: 2024-06-01 02:42:35.841718159 +0000 UTC m=+0.563813352 container cleanup 108dda5fe7cc0c23a6888f3c445309bfa86e8a0b0a51ab966affb69af2b19757 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_CLEAN=True, architecture=x86_64, version=8, vcs-type=git, ceph=True, release=754, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public) 2024-06-01T02:42:36.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:35 smithi003 bash[67192]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003 2024-06-01T02:42:36.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 podman[67215]: 2024-06-01 02:42:36.126142317 +0000 UTC m=+0.637351132 container remove 108dda5fe7cc0c23a6888f3c445309bfa86e8a0b0a51ab966affb69af2b19757 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, vendor=Red Hat, Inc., io.openshift.expose-services=, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vcs-type=git, RELEASE=HEAD, ceph=True, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-06-01T02:42:36.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi003.service: Deactivated successfully. 2024-06-01T02:42:36.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 systemd[1]: Stopped Ceph mon.smithi003 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:42:36.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi003.service: Consumed 24.507s CPU time. 2024-06-01T02:42:36.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 systemd[1]: Starting Ceph mon.smithi003 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:42:36.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 podman[67344]: 2024-06-01 02:42:36.628081358 +0000 UTC m=+0.018991559 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:42:36.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 podman[67344]: 2024-06-01 02:42:36.735427884 +0000 UTC m=+0.126338050 container create a3fb82b575ae1f33e56589242e4ea7b3134b5aadc6f38d9cd72649b1b2583a5d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240528, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:42:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 podman[67344]: 2024-06-01 02:42:36.971823722 +0000 UTC m=+0.362733924 container init a3fb82b575ae1f33e56589242e4ea7b3134b5aadc6f38d9cd72649b1b2583a5d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-06-01T02:42:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 podman[67344]: 2024-06-01 02:42:36.975412277 +0000 UTC m=+0.366322451 container start a3fb82b575ae1f33e56589242e4ea7b3134b5aadc6f38d9cd72649b1b2583a5d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD) 2024-06-01T02:42:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 ceph-mon[67363]: set uid:gid to 167:167 (ceph:ceph) 2024-06-01T02:42:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 ceph-mon[67363]: ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc), process ceph-mon, pid 2 2024-06-01T02:42:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:36 smithi003 ceph-mon[67363]: pidfile_write: ignore empty --pid-file 2024-06-01T02:42:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: load: jerasure load: lrc 2024-06-01T02:42:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: RocksDB version: 7.9.2 2024-06-01T02:42:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Git sha 0 2024-06-01T02:42:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Compile date 2024-05-30 15:42:47 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: DB SUMMARY 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: DB Session ID: EIJKHUA8QFE9XL1YVUDL 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: CURRENT file: CURRENT 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: IDENTITY file: IDENTITY 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1912 Bytes 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi003/store.db dir, Total Num: 1, files: 000045.sst 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi003/store.db: 000043.log size: 2942117 ; 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.error_if_exists: 0 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.create_if_missing: 0 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.paranoid_checks: 1 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-01T02:42:37.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-01T02:42:37.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.env: 0x55f134a56c60 2024-06-01T02:42:37.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.fs: PosixFileSystem 2024-06-01T02:42:37.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.info_log: 0x55f13593d9e0 2024-06-01T02:42:37.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_file_opening_threads: 16 2024-06-01T02:42:37.340 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.statistics: (nil) 2024-06-01T02:42:37.340 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.use_fsync: 0 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_log_file_size: 0 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.keep_log_file_num: 1000 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.recycle_log_file_num: 0 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.allow_fallocate: 1 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.allow_mmap_reads: 0 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.allow_mmap_writes: 0 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.use_direct_reads: 0 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.create_missing_column_families: 0 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.db_log_dir: 2024-06-01T02:42:37.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.wal_dir: 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.advise_random_on_open: 1 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.db_write_buffer_size: 0 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.write_buffer_manager: 0x55f13590e1e0 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-01T02:42:37.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.rate_limiter: (nil) 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.wal_recovery_mode: 2 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.enable_thread_tracking: 0 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.enable_pipelined_write: 0 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.unordered_write: 0 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.row_cache: None 2024-06-01T02:42:37.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.wal_filter: None 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.allow_ingest_behind: 0 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.two_write_queues: 0 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.manual_wal_flush: 0 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.wal_compression: 0 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.atomic_flush: 0 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-01T02:42:37.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.log_readahead_size: 0 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.best_efforts_recovery: 0 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.allow_data_in_errors: 0 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.db_host_id: __hostname__ 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_background_jobs: 2 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_background_compactions: -1 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_subcompactions: 1 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-01T02:42:37.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_total_wal_size: 0 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_open_files: -1 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bytes_per_sync: 0 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_readahead_size: 0 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_background_flushes: -1 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Compression algorithms supported: 2024-06-01T02:42:37.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: kZSTD supported: 0 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: kXpressCompression supported: 0 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: kLZ4HCCompression supported: 1 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: kZlibCompression supported: 1 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: kSnappyCompression supported: 1 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: kLZ4Compression supported: 1 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: kBZip2Compression supported: 0 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi003/store.db/MANIFEST-000015 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.merge_operator: 2024-06-01T02:42:37.347 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_filter: None 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_filter_factory: None 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.sst_partitioner_factory: None 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f13593dae0) 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: cache_index_and_filter_blocks: 1 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: pin_top_level_index_and_filter: 1 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: index_type: 0 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: data_block_index_type: 0 2024-06-01T02:42:37.348 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: index_shortening: 1 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: checksum: 4 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: no_block_cache: 0 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_cache: 0x55f13593f090 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_cache_name: BinnedLRUCache 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_cache_options: 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: capacity : 536870912 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: num_shard_bits : 4 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: strict_capacity_limit : 0 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: high_pri_pool_ratio: 0.000 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_cache_compressed: (nil) 2024-06-01T02:42:37.349 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: persistent_cache: (nil) 2024-06-01T02:42:37.350 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_size: 4096 2024-06-01T02:42:37.350 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_size_deviation: 10 2024-06-01T02:42:37.350 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_restart_interval: 16 2024-06-01T02:42:37.350 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: index_block_restart_interval: 1 2024-06-01T02:42:37.350 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: metadata_block_size: 4096 2024-06-01T02:42:37.350 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: partition_filters: 0 2024-06-01T02:42:37.350 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: use_delta_encoding: 1 2024-06-01T02:42:37.350 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: filter_policy: bloomfilter 2024-06-01T02:42:37.350 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: whole_key_filtering: 1 2024-06-01T02:42:37.351 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: verify_compression: 0 2024-06-01T02:42:37.351 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: read_amp_bytes_per_bit: 0 2024-06-01T02:42:37.351 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: format_version: 5 2024-06-01T02:42:37.351 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: enable_index_compression: 1 2024-06-01T02:42:37.351 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: block_align: 0 2024-06-01T02:42:37.351 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: max_auto_readahead_size: 262144 2024-06-01T02:42:37.351 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: prepopulate_block_cache: 0 2024-06-01T02:42:37.351 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: initial_auto_readahead_size: 8192 2024-06-01T02:42:37.351 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: num_file_reads_for_auto_readahead: 2 2024-06-01T02:42:37.352 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.write_buffer_size: 33554432 2024-06-01T02:42:37.352 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_write_buffer_number: 2 2024-06-01T02:42:37.352 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression: NoCompression 2024-06-01T02:42:37.352 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression: Disabled 2024-06-01T02:42:37.352 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.prefix_extractor: nullptr 2024-06-01T02:42:37.352 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-01T02:42:37.352 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.num_levels: 7 2024-06-01T02:42:37.353 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-01T02:42:37.353 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-01T02:42:37.353 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-01T02:42:37.353 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-01T02:42:37.353 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-01T02:42:37.353 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression_opts.level: 32767 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression_opts.strategy: 0 2024-06-01T02:42:37.354 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-01T02:42:37.355 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-01T02:42:37.355 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-01T02:42:37.355 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-01T02:42:37.355 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression_opts.enabled: false 2024-06-01T02:42:37.355 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-01T02:42:37.355 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-01T02:42:37.355 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-01T02:42:37.355 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-01T02:42:37.355 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.target_file_size_base: 67108864 2024-06-01T02:42:37.356 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-01T02:42:37.356 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-01T02:42:37.356 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-01T02:42:37.356 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-01T02:42:37.356 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-01T02:42:37.357 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-01T02:42:37.357 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-01T02:42:37.357 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-01T02:42:37.358 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-01T02:42:37.358 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-01T02:42:37.358 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-01T02:42:37.358 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-01T02:42:37.358 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-01T02:42:37.358 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-01T02:42:37.359 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.arena_block_size: 1048576 2024-06-01T02:42:37.359 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-01T02:42:37.359 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-01T02:42:37.359 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.disable_auto_compactions: 0 2024-06-01T02:42:37.360 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-01T02:42:37.360 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-01T02:42:37.360 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-01T02:42:37.360 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-01T02:42:37.360 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-01T02:42:37.361 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-01T02:42:37.361 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-01T02:42:37.361 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-01T02:42:37.361 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-01T02:42:37.361 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-01T02:42:37.361 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.table_properties_collectors: 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.inplace_update_support: 0 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.bloom_locality: 0 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.max_successive_merges: 0 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.paranoid_file_checks: 0 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.force_consistency_checks: 1 2024-06-01T02:42:37.362 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.report_bg_io_stats: 0 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.ttl: 2592000 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.enable_blob_files: false 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.min_blob_size: 0 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.blob_file_size: 268435456 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.blob_file_starting_level: 0 2024-06-01T02:42:37.363 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 45.sst 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi003/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 47, last_sequence is 20923, log_number is 43,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 43 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: fc496501-0003-4d45-a912-5886bd277f32 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717209757006250, "job": 1, "event": "recovery_started", "wal_files": [43]} 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #43 mode 2 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717209757014586, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 48, "file_size": 2656556, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 20924, "largest_seqno": 22220, "table_properties": {"data_size": 2650134, "index_size": 3931, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13684, "raw_average_key_size": 24, "raw_value_size": 2637526, "raw_average_value_size": 4627, "num_data_blocks": 180, "num_entries": 570, "num_filter_entries": 570, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1717209757, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "fc496501-0003-4d45-a912-5886bd277f32", "db_session_id": "EIJKHUA8QFE9XL1YVUDL", "orig_file_number": 48, "seqno_to_time_mapping": "N/A"}} 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717209757014686, "job": 1, "event": "recovery_finished"} 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/version_set.cc:5047] Creating manifest 50 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi003/store.db/000043.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-01T02:42:37.364 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f135a2e000 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: DB pointer 0x55f135a18000 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** DB Stats ** 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:42:37.365 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** Compaction Stats [default] ** 2024-06-01T02:42:37.366 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-01T02:42:37.366 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-01T02:42:37.366 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: L0 1/0 2.53 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 637.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-01T02:42:37.366 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: L6 1/0 7.09 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-01T02:42:37.366 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Sum 2/0 9.62 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 637.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-01T02:42:37.366 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 637.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** Compaction Stats [default] ** 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 637.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-01T02:42:37.367 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Cumulative compaction: 0.00 GB write, 233.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Interval compaction: 0.00 GB write, 233.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Block cache BinnedLRUCache@0x55f13593f090#2 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: starting mon.smithi003 rank 0 at public addrs [v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0] at bind addrs [v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi003 fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: mon.smithi003@-1(???) e2 preinit fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: mon.smithi003@-1(???).mds e17 new map 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: mon.smithi003@-1(???).mds e17 print_map 2024-06-01T02:42:37.368 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: e17 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: legacy client fscid: 1 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Filesystem 'foofs' (1) 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: fs_name foofs 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: epoch 17 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: created 2024-06-01T02:23:42.709129+0000 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: modified 2024-06-01T02:36:07.013115+0000 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: tableserver 0 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: root 0 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: session_timeout 60 2024-06-01T02:42:37.369 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: session_autoclose 300 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: max_file_size 1099511627776 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: max_xattr_size 65536 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: required_client_features {} 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: last_failure 0 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: last_failure_osd_epoch 0 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: max_mds 1 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: in 0 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: up {0=14506} 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: failed 2024-06-01T02:42:37.370 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: damaged 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: stopped 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: data_pools [3] 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: metadata_pool 2 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: inline_data disabled 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: balancer 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: bal_rank_mask -1 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: standby_count_wanted 1 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: qdb_cluster leader: 0 members: 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: [mds.foofs.smithi003.kexxec{0:14506} state up:active seq 187 join_fscid=1 addr [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] compat {c=[1],r=[1],i=[7ff]}] 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: Standby daemons: 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: 2024-06-01T02:42:37.371 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout: [mds.foofs.smithi027.xaykap{-1:24299} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] compat {c=[1],r=[1],i=[7ff]}] 2024-06-01T02:42:37.372 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: mon.smithi003@-1(???).osd e85 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-01T02:42:37.372 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: mon.smithi003@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-06-01T02:42:37.372 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: mon.smithi003@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-06-01T02:42:37.372 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: mon.smithi003@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-06-01T02:42:37.372 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 ceph-mon[67363]: mon.smithi003@-1(???).paxosservice(auth 1..33) refresh upgraded, format 0 -> 3 2024-06-01T02:42:37.372 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 bash[67344]: a3fb82b575ae1f33e56589242e4ea7b3134b5aadc6f38d9cd72649b1b2583a5d 2024-06-01T02:42:37.372 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:37 smithi003 systemd[1]: Started Ceph mon.smithi003 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:42:38.826 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:42:38.826 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:42:38.826 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: pgmap v21: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:38.826 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: pgmap v22: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:38.826 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: mon.smithi003 calling monitor election 2024-06-01T02:42:38.826 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: mon.smithi003 is new leader, mons smithi003,smithi027 in quorum (ranks 0,1) 2024-06-01T02:42:38.826 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: monmap epoch 2 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: last_changed 2024-06-01T02:18:41.208406+0000 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: created 2024-06-01T02:16:35.108780+0000 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: min_mon_release 17 (quincy) 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: election_strategy: 1 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: 0: [v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0] mon.smithi003 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: 1: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.smithi027 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: osdmap e85: 8 total, 8 up, 8 in 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: mgrmap e33: smithi003.njsqxz(active, since 39s), standbys: smithi027.tujskj 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: overall HEALTH_OK 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: from='mgr.14798 ' entity='' 2024-06-01T02:42:38.827 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:38 smithi027 ceph-mon[27271]: mgrmap e34: smithi003.njsqxz(active, since 39s), standbys: smithi027.tujskj 2024-06-01T02:42:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:42:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: from='mgr.14798 172.21.15.3:0/1830356323' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:42:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: pgmap v21: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: pgmap v22: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:42:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: mon.smithi003 calling monitor election 2024-06-01T02:42:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: mon.smithi003 is new leader, mons smithi003,smithi027 in quorum (ranks 0,1) 2024-06-01T02:42:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: monmap epoch 2 2024-06-01T02:42:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: last_changed 2024-06-01T02:18:41.208406+0000 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: created 2024-06-01T02:16:35.108780+0000 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: min_mon_release 17 (quincy) 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: election_strategy: 1 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: 0: [v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0] mon.smithi003 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: 1: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.smithi027 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: osdmap e85: 8 total, 8 up, 8 in 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: mgrmap e33: smithi003.njsqxz(active, since 39s), standbys: smithi027.tujskj 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: overall HEALTH_OK 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: from='mgr.14798 ' entity='' 2024-06-01T02:42:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:38 smithi003 ceph-mon[67363]: mgrmap e34: smithi003.njsqxz(active, since 39s), standbys: smithi027.tujskj 2024-06-01T02:42:40.777 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:40 smithi027 ceph-mon[27271]: mgrmap e35: smithi003.njsqxz(active, since 40s), standbys: smithi027.tujskj 2024-06-01T02:42:40.802 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:40 smithi003 ceph-mon[67363]: mgrmap e35: smithi003.njsqxz(active, since 40s), standbys: smithi027.tujskj 2024-06-01T02:42:45.802 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:45 smithi003 ceph-mon[67363]: Standby manager daemon smithi027.tujskj restarted 2024-06-01T02:42:45.802 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:45 smithi003 ceph-mon[67363]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:42:45.803 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:45 smithi003 ceph-mon[67363]: from='mgr.? 172.21.15.27:0/2269376115' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:42:45.803 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:45 smithi003 ceph-mon[67363]: from='mgr.? 172.21.15.27:0/2269376115' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:42:45.803 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:45 smithi003 ceph-mon[67363]: from='mgr.? 172.21.15.27:0/2269376115' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:42:45.803 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:45 smithi003 ceph-mon[67363]: from='mgr.? 172.21.15.27:0/2269376115' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:42:45.930 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:45 smithi027 ceph-mon[27271]: Standby manager daemon smithi027.tujskj restarted 2024-06-01T02:42:45.930 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:45 smithi027 ceph-mon[27271]: Standby manager daemon smithi027.tujskj started 2024-06-01T02:42:45.930 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:45 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/2269376115' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/crt"}]: dispatch 2024-06-01T02:42:45.930 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:45 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/2269376115' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-01T02:42:45.931 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:45 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/2269376115' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi027.tujskj/key"}]: dispatch 2024-06-01T02:42:45.931 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:45 smithi027 ceph-mon[27271]: from='mgr.? 172.21.15.27:0/2269376115' entity='mgr.smithi027.tujskj' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-01T02:42:46.754 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:42:46.925 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: mgrmap e36: smithi003.njsqxz(active, since 46s), standbys: smithi027.tujskj 2024-06-01T02:42:46.925 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: Active manager daemon smithi003.njsqxz restarted 2024-06-01T02:42:46.925 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: Activating manager daemon smithi003.njsqxz 2024-06-01T02:42:46.925 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: osdmap e86: 8 total, 8 up, 8 in 2024-06-01T02:42:46.925 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: mgrmap e37: smithi003.njsqxz(active, starting, since 0.00572428s), standbys: smithi027.tujskj 2024-06-01T02:42:46.925 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:42:46.925 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:42:46.925 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi027.tujskj", "id": "smithi027.tujskj"}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:42:46.926 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:42:46.927 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:42:46.927 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: Manager daemon smithi003.njsqxz is now available 2024-06-01T02:42:46.927 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:46.927 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:42:46.927 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:42:46.927 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:42:46.927 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: [01/Jun/2024:02:42:46] ENGINE Bus STARTING 2024-06-01T02:42:46.927 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:46 smithi027 ceph-mon[27271]: [01/Jun/2024:02:42:46] ENGINE Serving on http://172.21.15.3:8765 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: mgrmap e36: smithi003.njsqxz(active, since 46s), standbys: smithi027.tujskj 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: Active manager daemon smithi003.njsqxz restarted 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: Activating manager daemon smithi003.njsqxz 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: osdmap e86: 8 total, 8 up, 8 in 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: mgrmap e37: smithi003.njsqxz(active, starting, since 0.00572428s), standbys: smithi027.tujskj 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:42:46.953 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi003.njsqxz", "id": "smithi003.njsqxz"}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr metadata", "who": "smithi027.tujskj", "id": "smithi027.tujskj"}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-01T02:42:46.954 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: Manager daemon smithi003.njsqxz is now available 2024-06-01T02:42:46.955 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:46.955 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:42:46.955 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/mirror_snapshot_schedule"}]: dispatch 2024-06-01T02:42:46.955 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi003.njsqxz/trash_purge_schedule"}]: dispatch 2024-06-01T02:42:46.955 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: [01/Jun/2024:02:42:46] ENGINE Bus STARTING 2024-06-01T02:42:46.955 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:46 smithi003 ceph-mon[67363]: [01/Jun/2024:02:42:46] ENGINE Serving on http://172.21.15.3:8765 2024-06-01T02:42:47.466 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:42:47.466 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (78s) 19s ago 24m 13.5M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:42:47.466 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (24m) 19s ago 24m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:42:47.466 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (24m) 20s ago 24m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:42:47.466 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (74s) 19s ago 24m 45.0M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:42:47.466 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (19m) 19s ago 19m 40.3M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (18m) 20s ago 18m 17.8M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (59s) 19s ago 26m 495M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (25s) 20s ago 24m 113M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (26m) 19s ago 26m 68.9M 2048M 17.2.0 e1d6a67b021e 108dda5fe7cc 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (24m) 20s ago 24m 51.2M 2048M 17.2.0 e1d6a67b021e edc537c54a3a 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (18m) 19s ago 18m 269M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (18m) 20s ago 18m 87.5M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (72s) 19s ago 24m 8396k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (65s) 20s ago 23m 8669k - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (23m) 19s ago 23m 442M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (22m) 19s ago 22m 360M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (22m) 19s ago 22m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (22m) 19s ago 22m 370M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (21m) 20s ago 21m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:42:47.467 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (21m) 20s ago 21m 459M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:42:47.468 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (20m) 20s ago 20m 375M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:42:47.468 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (20m) 20s ago 20m 617M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:42:47.468 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (31s) 19s ago 23m 51.0M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:42:47.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:47 smithi027 ceph-mon[27271]: [01/Jun/2024:02:42:46] ENGINE Serving on https://172.21.15.3:7150 2024-06-01T02:42:47.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:47 smithi027 ceph-mon[27271]: [01/Jun/2024:02:42:46] ENGINE Bus STARTED 2024-06-01T02:42:47.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:47 smithi027 ceph-mon[27271]: mgrmap e38: smithi003.njsqxz(active, since 1.0104s), standbys: smithi027.tujskj 2024-06-01T02:42:47.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:47 smithi027 ceph-mon[27271]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:47.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:47 smithi027 ceph-mon[27271]: pgmap v3: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:47.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:47 smithi027 ceph-mon[27271]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:47.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:47 smithi027 ceph-mon[27271]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:47.899 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:42:47.900 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:42:47.900 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-01T02:42:47.900 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 1 2024-06-01T02:42:47.900 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:42:47.900 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:42:47.900 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:42:47.900 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:42:47.900 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:42:47.900 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:42:47.901 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:42:47.901 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:42:47.901 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:42:47.901 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:42:47.901 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:42:47.901 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-06-01T02:42:47.901 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 3 2024-06-01T02:42:47.901 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:42:47.901 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:42:48.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:47 smithi003 ceph-mon[67363]: [01/Jun/2024:02:42:46] ENGINE Serving on https://172.21.15.3:7150 2024-06-01T02:42:48.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:47 smithi003 ceph-mon[67363]: [01/Jun/2024:02:42:46] ENGINE Bus STARTED 2024-06-01T02:42:48.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:47 smithi003 ceph-mon[67363]: mgrmap e38: smithi003.njsqxz(active, since 1.0104s), standbys: smithi027.tujskj 2024-06-01T02:42:48.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:47 smithi003 ceph-mon[67363]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:48.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:47 smithi003 ceph-mon[67363]: pgmap v3: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:48.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:47 smithi003 ceph-mon[67363]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:48.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:47 smithi003 ceph-mon[67363]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:48.270 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:42:48.270 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:42:48.270 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:42:48.270 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:42:48.270 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:42:48.270 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:42:48.270 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:42:48.270 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "2/23 daemons upgraded", 2024-06-01T02:42:48.271 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "", 2024-06-01T02:42:48.271 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:42:48.271 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:42:48.689 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:42:48.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:48 smithi003 ceph-mon[67363]: pgmap v4: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:48.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:48 smithi003 ceph-mon[67363]: mgrmap e39: smithi003.njsqxz(active, since 2s), standbys: smithi027.tujskj 2024-06-01T02:42:48.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:48 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1396894138' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:42:48.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:48 smithi003 ceph-mon[67363]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:48.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:48 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/2724020234' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:42:48.851 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:48 smithi027 ceph-mon[27271]: pgmap v4: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:48.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:48 smithi027 ceph-mon[27271]: mgrmap e39: smithi003.njsqxz(active, since 2s), standbys: smithi027.tujskj 2024-06-01T02:42:48.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:48 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/1396894138' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:42:48.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:48 smithi027 ceph-mon[27271]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:42:48.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:48 smithi027 ceph-mon[27271]: from='client.? 172.21.15.3:0/2724020234' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:42:50.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:50 smithi003 ceph-mon[67363]: pgmap v5: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:50.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:50 smithi003 ceph-mon[67363]: mgrmap e40: smithi003.njsqxz(active, since 4s), standbys: smithi027.tujskj 2024-06-01T02:42:50.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:50 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:50.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:50 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:51.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:50 smithi027 ceph-mon[27271]: pgmap v5: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:51.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:50 smithi027 ceph-mon[27271]: mgrmap e40: smithi003.njsqxz(active, since 4s), standbys: smithi027.tujskj 2024-06-01T02:42:51.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:50 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:51.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:50 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:52.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:52.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:52.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:52.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:52 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:52 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:52 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:52 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:53.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:53 smithi003 ceph-mon[67363]: pgmap v6: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:53.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:53 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:53.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:53 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:53.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:53 smithi027 ceph-mon[27271]: pgmap v6: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:42:53.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:53 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:53.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:53 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:55.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:55 smithi003 ceph-mon[67363]: pgmap v7: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-01T02:42:55.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:55.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:42:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:55 smithi027 ceph-mon[27271]: pgmap v7: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-01T02:42:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:55 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:55 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:55 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:55 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:42:56.547 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:42:56.547 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-01T02:42:56.547 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:42:56.547 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:42:56.547 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: Updating smithi003:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:42:56.547 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:42:56.547 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:56.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:42:56.548 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:56 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-01T02:42:56.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: Updating smithi003:/etc/ceph/ceph.conf 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: Updating smithi027:/etc/ceph/ceph.conf 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: Updating smithi003:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:42:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:42:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-01T02:42:57.549 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:57 smithi027 ceph-mon[27271]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:42:57.550 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:57 smithi027 ceph-mon[27271]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:42:57.550 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:57 smithi027 ceph-mon[27271]: pgmap v8: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-01T02:42:57.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:57 smithi003 ceph-mon[67363]: Updating smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:42:57.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:57 smithi003 ceph-mon[67363]: Updating smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-01T02:42:57.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:57 smithi003 ceph-mon[67363]: pgmap v8: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-01T02:42:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:58 smithi027 ceph-mon[27271]: Upgrade: Updating mon.smithi027 2024-06-01T02:42:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:58 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:58 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:42:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:58 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:42:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:58 smithi027 ceph-mon[27271]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:58 smithi027 ceph-mon[27271]: Deploying daemon mon.smithi027 on smithi027 2024-06-01T02:42:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:58 smithi003 ceph-mon[67363]: Upgrade: Updating mon.smithi027 2024-06-01T02:42:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:42:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:42:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:42:58.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:42:58.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:58 smithi003 ceph-mon[67363]: Deploying daemon mon.smithi027 on smithi027 2024-06-01T02:42:59.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:59 smithi027 systemd[1]: Stopping Ceph mon.smithi027 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:42:59.764 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:59 smithi027 ceph-mon[27271]: pgmap v9: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-01T02:42:59.765 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027[27264]: 2024-06-01T02:42:59.463+0000 7f8dbb74f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi027 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:42:59.765 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027[27264]: 2024-06-01T02:42:59.463+0000 7f8dbb74f700 -1 mon.smithi027@1(peon) e2 *** Got Signal Terminated *** 2024-06-01T02:42:59.765 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:59 smithi027 podman[53645]: 2024-06-01 02:42:59.495569347 +0000 UTC m=+0.165370030 container died edc537c54a3ad9f24f56782b13871c4c220ec52f067649c1356d66c541103ede (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, RELEASE=HEAD, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, name=centos-stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-06-01T02:42:59.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:42:59 smithi003 ceph-mon[67363]: pgmap v9: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-01T02:43:00.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:59 smithi027 podman[53645]: 2024-06-01 02:42:59.864966414 +0000 UTC m=+0.534767097 container cleanup edc537c54a3ad9f24f56782b13871c4c220ec52f067649c1356d66c541103ede (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, ceph=True, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container) 2024-06-01T02:43:00.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:42:59 smithi027 bash[53645]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027 2024-06-01T02:43:00.502 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:00 smithi027 podman[53660]: 2024-06-01 02:43:00.124509524 +0000 UTC m=+0.628885382 container remove edc537c54a3ad9f24f56782b13871c4c220ec52f067649c1356d66c541103ede (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, distribution-scope=public, vcs-type=git, vendor=Red Hat, Inc., RELEASE=HEAD, architecture=x86_64, GIT_BRANCH=HEAD, name=centos-stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-06-01T02:43:00.503 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:00 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi027.service: Deactivated successfully. 2024-06-01T02:43:00.503 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:00 smithi027 systemd[1]: Stopped Ceph mon.smithi027 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:43:00.503 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:00 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi027.service: Consumed 11.114s CPU time. 2024-06-01T02:43:00.828 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:00 smithi027 systemd[1]: Starting Ceph mon.smithi027 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:43:00.828 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:00 smithi027 podman[53782]: 2024-06-01 02:43:00.625522087 +0000 UTC m=+0.018090146 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:43:00.829 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:00 smithi027 podman[53782]: 2024-06-01 02:43:00.7257781 +0000 UTC m=+0.118346131 container create f58a9b5236e40de221ae0ff0acb4b020858c929726a6ddfd6d70cff9e36f9778 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:43:01.080 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 podman[53782]: 2024-06-01 02:43:01.078333625 +0000 UTC m=+0.470901671 container init f58a9b5236e40de221ae0ff0acb4b020858c929726a6ddfd6d70cff9e36f9778 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True) 2024-06-01T02:43:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 podman[53782]: 2024-06-01 02:43:01.081728962 +0000 UTC m=+0.474296996 container start f58a9b5236e40de221ae0ff0acb4b020858c929726a6ddfd6d70cff9e36f9778 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-3d89acc) 2024-06-01T02:43:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: set uid:gid to 167:167 (ceph:ceph) 2024-06-01T02:43:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc), process ceph-mon, pid 2 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: pidfile_write: ignore empty --pid-file 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: load: jerasure load: lrc 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: RocksDB version: 7.9.2 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Git sha 0 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Compile date 2024-05-30 15:42:47 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: DB SUMMARY 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: DB Session ID: NN3CH61D4JB3YQQSXUSR 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: CURRENT file: CURRENT 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: IDENTITY file: IDENTITY 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2225 Bytes 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi027/store.db dir, Total Num: 1, files: 000045.sst 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi027/store.db: 000043.log size: 36285 ; 2024-06-01T02:43:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.error_if_exists: 0 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.create_if_missing: 0 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.paranoid_checks: 1 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.env: 0x558367c63c60 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.fs: PosixFileSystem 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.info_log: 0x558368aa59e0 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_file_opening_threads: 16 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.statistics: (nil) 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.use_fsync: 0 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_log_file_size: 0 2024-06-01T02:43:01.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-01T02:43:01.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-01T02:43:01.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.keep_log_file_num: 1000 2024-06-01T02:43:01.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.recycle_log_file_num: 0 2024-06-01T02:43:01.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.allow_fallocate: 1 2024-06-01T02:43:01.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.allow_mmap_reads: 0 2024-06-01T02:43:01.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.allow_mmap_writes: 0 2024-06-01T02:43:01.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.use_direct_reads: 0 2024-06-01T02:43:01.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-01T02:43:01.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.create_missing_column_families: 0 2024-06-01T02:43:01.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.db_log_dir: 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.wal_dir: 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.advise_random_on_open: 1 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.db_write_buffer_size: 0 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.write_buffer_manager: 0x558368a761e0 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-01T02:43:01.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.rate_limiter: (nil) 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.wal_recovery_mode: 2 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.enable_thread_tracking: 0 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.enable_pipelined_write: 0 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.unordered_write: 0 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.row_cache: None 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.wal_filter: None 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.allow_ingest_behind: 0 2024-06-01T02:43:01.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.two_write_queues: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.manual_wal_flush: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.wal_compression: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.atomic_flush: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.log_readahead_size: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.best_efforts_recovery: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.allow_data_in_errors: 0 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.db_host_id: __hostname__ 2024-06-01T02:43:01.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_background_jobs: 2 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_background_compactions: -1 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_subcompactions: 1 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_total_wal_size: 0 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_open_files: -1 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bytes_per_sync: 0 2024-06-01T02:43:01.360 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_readahead_size: 0 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_background_flushes: -1 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Compression algorithms supported: 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: kZSTD supported: 0 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: kXpressCompression supported: 0 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: kLZ4HCCompression supported: 1 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: kZlibCompression supported: 1 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: kSnappyCompression supported: 1 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: kLZ4Compression supported: 1 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: kBZip2Compression supported: 0 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-01T02:43:01.361 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi027/store.db/MANIFEST-000009 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.merge_operator: 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_filter: None 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_filter_factory: None 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.sst_partitioner_factory: None 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558368aa5ae0) 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-01T02:43:01.362 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: index_type: 0 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: data_block_index_type: 0 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: index_shortening: 1 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: checksum: 4 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: no_block_cache: 0 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_cache: 0x558368aa7090 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_cache_options: 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: capacity : 536870912 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: num_shard_bits : 4 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: strict_capacity_limit : 0 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-06-01T02:43:01.363 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_cache_compressed: (nil) 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: persistent_cache: (nil) 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_size: 4096 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_size_deviation: 10 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_restart_interval: 16 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: index_block_restart_interval: 1 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: metadata_block_size: 4096 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: partition_filters: 0 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: use_delta_encoding: 1 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: filter_policy: bloomfilter 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: whole_key_filtering: 1 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: verify_compression: 0 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: format_version: 5 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: enable_index_compression: 1 2024-06-01T02:43:01.364 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: block_align: 0 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: max_auto_readahead_size: 262144 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: prepopulate_block_cache: 0 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: initial_auto_readahead_size: 8192 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.write_buffer_size: 33554432 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_write_buffer_number: 2 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression: NoCompression 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression: Disabled 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.prefix_extractor: nullptr 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.num_levels: 7 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-01T02:43:01.365 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression_opts.level: 32767 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression_opts.strategy: 0 2024-06-01T02:43:01.366 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression_opts.enabled: false 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.target_file_size_base: 67108864 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-01T02:43:01.367 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.arena_block_size: 1048576 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.disable_auto_compactions: 0 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-01T02:43:01.368 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.table_properties_collectors: 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.inplace_update_support: 0 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-01T02:43:01.369 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.bloom_locality: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.max_successive_merges: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.paranoid_file_checks: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.force_consistency_checks: 1 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.report_bg_io_stats: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.ttl: 2592000 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.enable_blob_files: false 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.min_blob_size: 0 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.blob_file_size: 268435456 2024-06-01T02:43:01.370 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.blob_file_starting_level: 0 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 45.sst 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi027/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 47, last_sequence is 24813, log_number is 43,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 43 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 19bf49a4-ecec-48b2-80ab-f910aa86e7f1 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717209781113788, "job": 1, "event": "recovery_started", "wal_files": [43]} 2024-06-01T02:43:01.371 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #43 mode 2 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717209781114710, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 48, "file_size": 24895, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 24818, "largest_seqno": 24864, "table_properties": {"data_size": 23667, "index_size": 89, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 133, "raw_key_size": 867, "raw_average_key_size": 27, "raw_value_size": 22909, "raw_average_value_size": 715, "num_data_blocks": 4, "num_entries": 32, "num_filter_entries": 32, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1717209781, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "19bf49a4-ecec-48b2-80ab-f910aa86e7f1", "db_session_id": "NN3CH61D4JB3YQQSXUSR", "orig_file_number": 48, "seqno_to_time_mapping": "N/A"}} 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: EVENT_LOG_v1 {"time_micros": 1717209781114821, "job": 1, "event": "recovery_finished"} 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/version_set.cc:5047] Creating manifest 50 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi027/store.db/000043.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x558368b96000 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: DB pointer 0x558368b80000 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** DB Stats ** 2024-06-01T02:43:01.373 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-01T02:43:01.374 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: L0 1/0 24.31 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 44.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: L6 1/0 10.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Sum 2/0 10.36 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 44.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 44.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** Compaction Stats [default] ** 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 44.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-01T02:43:01.375 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Cumulative compaction: 0.00 GB write, 6.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Interval compaction: 0.00 GB write, 6.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Block cache BinnedLRUCache@0x558368aa7090#2 capacity: 512.00 MB usage: 0.34 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.17 KB,3.27826e-05%) IndexBlock(1,0.17 KB,3.27826e-05%) Misc(1,0.00 KB,0%) 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: starting mon.smithi027 rank 1 at public addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] at bind addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi027 fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: mon.smithi027@-1(???) e2 preinit fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: mon.smithi027@-1(???).mds e17 new map 2024-06-01T02:43:01.376 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: mon.smithi027@-1(???).mds e17 print_map 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: e17 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: legacy client fscid: 1 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Filesystem 'foofs' (1) 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: fs_name foofs 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: epoch 17 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: created 2024-06-01T02:23:42.709129+0000 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: modified 2024-06-01T02:36:07.013115+0000 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: tableserver 0 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: root 0 2024-06-01T02:43:01.377 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: session_timeout 60 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: session_autoclose 300 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: max_file_size 1099511627776 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: max_xattr_size 65536 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: required_client_features {} 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: last_failure 0 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: last_failure_osd_epoch 0 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: max_mds 1 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: in 0 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: up {0=14506} 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: failed 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: damaged 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: stopped 2024-06-01T02:43:01.378 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: data_pools [3] 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: metadata_pool 2 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: inline_data disabled 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: balancer 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: bal_rank_mask -1 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: standby_count_wanted 1 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: qdb_cluster leader: 0 members: 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: [mds.foofs.smithi003.kexxec{0:14506} state up:active seq 187 join_fscid=1 addr [v2:172.21.15.3:6834/4277191021,v1:172.21.15.3:6835/4277191021] compat {c=[1],r=[1],i=[7ff]}] 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: Standby daemons: 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout: [mds.foofs.smithi027.xaykap{-1:24299} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] compat {c=[1],r=[1],i=[7ff]}] 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: mon.smithi027@-1(???).osd e86 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-01T02:43:01.379 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: mon.smithi027@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2024-06-01T02:43:01.380 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: mon.smithi027@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2024-06-01T02:43:01.380 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: mon.smithi027@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2024-06-01T02:43:01.380 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 ceph-mon[53801]: mon.smithi027@-1(???).paxosservice(auth 1..34) refresh upgraded, format 0 -> 3 2024-06-01T02:43:01.380 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 bash[53782]: f58a9b5236e40de221ae0ff0acb4b020858c929726a6ddfd6d70cff9e36f9778 2024-06-01T02:43:01.380 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:01 smithi027 systemd[1]: Started Ceph mon.smithi027 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:43:03.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: pgmap v11: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:43:03.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: mon.smithi003 calling monitor election 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: mon.smithi027 calling monitor election 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: mon.smithi003 is new leader, mons smithi003,smithi027 in quorum (ranks 0,1) 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: monmap epoch 3 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: last_changed 2024-06-01T02:43:01.846737+0000 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: created 2024-06-01T02:16:35.108780+0000 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: min_mon_release 19 (squid) 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: election_strategy: 1 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: 0: [v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0] mon.smithi003 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: 1: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.smithi027 2024-06-01T02:43:03.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:43:03.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: osdmap e86: 8 total, 8 up, 8 in 2024-06-01T02:43:03.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: mgrmap e40: smithi003.njsqxz(active, since 16s), standbys: smithi027.tujskj 2024-06-01T02:43:03.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: overall HEALTH_OK 2024-06-01T02:43:03.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:03.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:03.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:02 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:43:03.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: pgmap v11: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi003"}]: dispatch 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mon metadata", "id": "smithi027"}]: dispatch 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: mon.smithi003 calling monitor election 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: mon.smithi027 calling monitor election 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: mon.smithi003 is new leader, mons smithi003,smithi027 in quorum (ranks 0,1) 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: monmap epoch 3 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: last_changed 2024-06-01T02:43:01.846737+0000 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: created 2024-06-01T02:16:35.108780+0000 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: min_mon_release 19 (squid) 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: election_strategy: 1 2024-06-01T02:43:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: 0: [v2:172.21.15.3:3300/0,v1:172.21.15.3:6789/0] mon.smithi003 2024-06-01T02:43:03.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: 1: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.smithi027 2024-06-01T02:43:03.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:43:03.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: osdmap e86: 8 total, 8 up, 8 in 2024-06-01T02:43:03.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: mgrmap e40: smithi003.njsqxz(active, since 16s), standbys: smithi027.tujskj 2024-06-01T02:43:03.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: overall HEALTH_OK 2024-06-01T02:43:03.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:03.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:03.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:02 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:43:05.158 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:04 smithi027 ceph-mon[53801]: pgmap v12: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:43:05.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:04 smithi003 ceph-mon[67363]: pgmap v12: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-01T02:43:07.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:06 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:07.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:06 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:07.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:06 smithi003 ceph-mon[67363]: pgmap v13: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:07.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:06 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:07.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:06 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:07.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:06 smithi027 ceph-mon[53801]: pgmap v13: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:08.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:07 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:08.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:07 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:08.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:07 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:08.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:07 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:08.945 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:08 smithi027 ceph-mon[53801]: pgmap v14: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:09.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:08 smithi003 ceph-mon[67363]: pgmap v14: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:11.712 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: pgmap v15: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:11.712 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:11.712 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:11.712 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:11.712 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:43:11.712 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:11.712 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:11.713 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:43:11.713 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:43:11.713 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: pgmap v15: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:43:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:11.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:11.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:43:11.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:43:11.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:12.732 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:12 smithi003 ceph-mon[67363]: Reconfiguring mon.smithi003 (monmap changed)... 2024-06-01T02:43:12.732 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:12 smithi003 ceph-mon[67363]: Reconfiguring daemon mon.smithi003 on smithi003 2024-06-01T02:43:12.732 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:12.732 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:12.732 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi003.njsqxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:43:12.732 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:43:12.732 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:12 smithi027 ceph-mon[53801]: Reconfiguring mon.smithi003 (monmap changed)... 2024-06-01T02:43:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:12 smithi027 ceph-mon[53801]: Reconfiguring daemon mon.smithi003 on smithi003 2024-06-01T02:43:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:12.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:12.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi003.njsqxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:43:12.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:43:12.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:13.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:13 smithi003 ceph-mon[67363]: pgmap v16: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:13.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:13 smithi003 ceph-mon[67363]: Reconfiguring mgr.smithi003.njsqxz (monmap changed)... 2024-06-01T02:43:13.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:13 smithi003 ceph-mon[67363]: Reconfiguring daemon mgr.smithi003.njsqxz on smithi003 2024-06-01T02:43:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:13 smithi027 ceph-mon[53801]: pgmap v16: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:13 smithi027 ceph-mon[53801]: Reconfiguring mgr.smithi003.njsqxz (monmap changed)... 2024-06-01T02:43:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:13 smithi027 ceph-mon[53801]: Reconfiguring daemon mgr.smithi003.njsqxz on smithi003 2024-06-01T02:43:15.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:14 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:15.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:14 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:15.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:14 smithi003 ceph-mon[67363]: pgmap v17: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:15.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:14 smithi003 ceph-mon[67363]: Reconfiguring crash.smithi003 (monmap changed)... 2024-06-01T02:43:15.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:14 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi003", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:43:15.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:14 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:15.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:14 smithi003 ceph-mon[67363]: Reconfiguring daemon crash.smithi003 on smithi003 2024-06-01T02:43:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:14 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:14 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:14 smithi027 ceph-mon[53801]: pgmap v17: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:14 smithi027 ceph-mon[53801]: Reconfiguring crash.smithi003 (monmap changed)... 2024-06-01T02:43:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:14 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi003", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:43:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:14 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:14 smithi027 ceph-mon[53801]: Reconfiguring daemon crash.smithi003 on smithi003 2024-06-01T02:43:16.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:16.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:16.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:16 smithi003 ceph-mon[67363]: Reconfiguring osd.0 (monmap changed)... 2024-06-01T02:43:16.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-01T02:43:16.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:16.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:16 smithi003 ceph-mon[67363]: Reconfiguring daemon osd.0 on smithi003 2024-06-01T02:43:16.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:43:16.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:16.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:16.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:16 smithi027 ceph-mon[53801]: Reconfiguring osd.0 (monmap changed)... 2024-06-01T02:43:16.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-01T02:43:16.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:16.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:16 smithi027 ceph-mon[53801]: Reconfiguring daemon osd.0 on smithi003 2024-06-01T02:43:16.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:43:17.688 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:17 smithi003 ceph-mon[67363]: pgmap v18: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:17.688 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:17 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:17.688 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:17 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:17.689 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:17 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-01T02:43:17.689 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:17 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:17.822 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:17 smithi027 ceph-mon[53801]: pgmap v18: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:17.822 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:17 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:17.822 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:17 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:17.822 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:17 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-01T02:43:17.822 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:17 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:18.618 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:18 smithi003 ceph-mon[67363]: Reconfiguring osd.1 (monmap changed)... 2024-06-01T02:43:18.619 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:18 smithi003 ceph-mon[67363]: Reconfiguring daemon osd.1 on smithi003 2024-06-01T02:43:18.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:18 smithi027 ceph-mon[53801]: Reconfiguring osd.1 (monmap changed)... 2024-06-01T02:43:18.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:18 smithi027 ceph-mon[53801]: Reconfiguring daemon osd.1 on smithi003 2024-06-01T02:43:19.063 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:43:19.663 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:19 smithi003 ceph-mon[67363]: pgmap v19: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:19.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:19 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:19.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:19 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:19.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:19 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-01T02:43:19.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:19 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:19.764 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:43:19.764 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (111s) 27s ago 25m 13.5M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:43:19.764 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (25m) 27s ago 25m 6409k - 17.2.0 e1d6a67b021e 28b2d012b583 2024-06-01T02:43:19.764 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (24m) 14s ago 24m 6405k - 17.2.0 e1d6a67b021e 3c3ea476ee58 2024-06-01T02:43:19.764 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (106s) 27s ago 25m 44.1M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:43:19.764 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (19m) 27s ago 19m 40.6M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (19m) 14s ago 19m 18.1M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (92s) 27s ago 26m 476M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (58s) 14s ago 24m 449M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (43s) 27s ago 26m 29.7M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (19s) 14s ago 24m 18.0M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (19m) 27s ago 19m 278M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (19m) 14s ago 19m 88.1M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (104s) 27s ago 25m 8660k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (97s) 14s ago 24m 9424k - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (24m) 27s ago 24m 442M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (23m) 27s ago 23m 360M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (23m) 27s ago 23m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:43:19.765 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (22m) 27s ago 22m 370M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:43:19.766 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (22m) 14s ago 22m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:43:19.766 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (21m) 14s ago 21m 460M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:43:19.766 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (21m) 14s ago 21m 375M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:43:19.766 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (21m) 14s ago 21m 617M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:43:19.766 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (63s) 27s ago 24m 58.0M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:43:19.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:19 smithi027 ceph-mon[53801]: pgmap v19: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:19.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:19 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:19.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:19 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:19.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:19 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-01T02:43:19.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:19 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:20.181 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:43:20.181 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:43:20.181 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:43:20.181 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:43:20.181 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:43:20.182 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-06-01T02:43:20.183 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 4 2024-06-01T02:43:20.183 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:43:20.183 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:43:20.573 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:43:20.573 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:43:20.573 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:43:20.573 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:43:20.574 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:43:20.574 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:43:20.574 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:43:20.574 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:43:20.574 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "4/23 daemons upgraded", 2024-06-01T02:43:20.574 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading mon daemons", 2024-06-01T02:43:20.575 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:43:20.575 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:43:20.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:20 smithi003 ceph-mon[67363]: Reconfiguring osd.2 (monmap changed)... 2024-06-01T02:43:20.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:20 smithi003 ceph-mon[67363]: Reconfiguring daemon osd.2 on smithi003 2024-06-01T02:43:20.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:20 smithi003 ceph-mon[67363]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:20.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:20 smithi003 ceph-mon[67363]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:20.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:20 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3221129137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:20.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:20 smithi027 ceph-mon[53801]: Reconfiguring osd.2 (monmap changed)... 2024-06-01T02:43:20.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:20 smithi027 ceph-mon[53801]: Reconfiguring daemon osd.2 on smithi003 2024-06-01T02:43:20.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:20 smithi027 ceph-mon[53801]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:20.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:20 smithi027 ceph-mon[53801]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:20.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:20 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3221129137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:21.010 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:43:21.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:21 smithi003 ceph-mon[67363]: pgmap v20: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:21.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:21 smithi003 ceph-mon[67363]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:21.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:21.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:21.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-01T02:43:21.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:21.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:21 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3202044105' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:43:21.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:21 smithi027 ceph-mon[53801]: pgmap v20: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:21.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:21 smithi027 ceph-mon[53801]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:21.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:21.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:21.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-01T02:43:21.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:21 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3202044105' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:43:22.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:22 smithi003 ceph-mon[67363]: Reconfiguring osd.3 (monmap changed)... 2024-06-01T02:43:22.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:22 smithi003 ceph-mon[67363]: Reconfiguring daemon osd.3 on smithi003 2024-06-01T02:43:22.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:22 smithi003 ceph-mon[67363]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:22.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:22.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:22.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi003.kexxec", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:43:22.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:22.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:22 smithi027 ceph-mon[53801]: Reconfiguring osd.3 (monmap changed)... 2024-06-01T02:43:22.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:22 smithi027 ceph-mon[53801]: Reconfiguring daemon osd.3 on smithi003 2024-06-01T02:43:22.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:22 smithi027 ceph-mon[53801]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:22.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:22.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:22.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi003.kexxec", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:43:22.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:23.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:23 smithi003 ceph-mon[67363]: pgmap v21: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:23.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:23 smithi003 ceph-mon[67363]: Reconfiguring mds.foofs.smithi003.kexxec (monmap changed)... 2024-06-01T02:43:23.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:23 smithi003 ceph-mon[67363]: Reconfiguring daemon mds.foofs.smithi003.kexxec on smithi003 2024-06-01T02:43:23.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:23 smithi027 ceph-mon[53801]: pgmap v21: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:23.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:23 smithi027 ceph-mon[53801]: Reconfiguring mds.foofs.smithi003.kexxec (monmap changed)... 2024-06-01T02:43:23.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:23 smithi027 ceph-mon[53801]: Reconfiguring daemon mds.foofs.smithi003.kexxec on smithi003 2024-06-01T02:43:24.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:24 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:24.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:24 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:24.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:24 smithi027 ceph-mon[53801]: Reconfiguring crash.smithi027 (monmap changed)... 2024-06-01T02:43:24.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:24 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:43:24.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:24 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:24.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:24 smithi027 ceph-mon[53801]: Reconfiguring daemon crash.smithi027 on smithi027 2024-06-01T02:43:24.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:24 smithi027 ceph-mon[53801]: pgmap v22: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:25.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:24 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:24 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:24 smithi003 ceph-mon[67363]: Reconfiguring crash.smithi027 (monmap changed)... 2024-06-01T02:43:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:24 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:43:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:24 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:24 smithi003 ceph-mon[67363]: Reconfiguring daemon crash.smithi027 on smithi027 2024-06-01T02:43:25.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:24 smithi003 ceph-mon[67363]: pgmap v22: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:26 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:26 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:26 smithi027 ceph-mon[53801]: Reconfiguring mgr.smithi027.tujskj (monmap changed)... 2024-06-01T02:43:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:26 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:43:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:26 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:43:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:26 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:26 smithi027 ceph-mon[53801]: Reconfiguring daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:43:26.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:26 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:26.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:26 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:26.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:26 smithi003 ceph-mon[67363]: Reconfiguring mgr.smithi027.tujskj (monmap changed)... 2024-06-01T02:43:26.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:26 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi027.tujskj", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-01T02:43:26.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:26 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-01T02:43:26.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:26 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:26.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:26 smithi003 ceph-mon[67363]: Reconfiguring daemon mgr.smithi027.tujskj on smithi027 2024-06-01T02:43:27.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:27 smithi027 ceph-mon[53801]: pgmap v23: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:27.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:27 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:27.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:27 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:27.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:27 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:43:27.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:27 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:43:27.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:27 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:27.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:27 smithi003 ceph-mon[67363]: pgmap v23: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:27.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:27 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:27 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:27 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-01T02:43:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:27 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-01T02:43:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:27 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:28.660 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:28 smithi003 ceph-mon[67363]: Reconfiguring mon.smithi027 (monmap changed)... 2024-06-01T02:43:28.660 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:28 smithi003 ceph-mon[67363]: Reconfiguring daemon mon.smithi027 on smithi027 2024-06-01T02:43:28.713 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:28 smithi027 ceph-mon[53801]: Reconfiguring mon.smithi027 (monmap changed)... 2024-06-01T02:43:28.713 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:28 smithi027 ceph-mon[53801]: Reconfiguring daemon mon.smithi027 on smithi027 2024-06-01T02:43:29.632 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:29 smithi027 ceph-mon[53801]: pgmap v24: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:29.632 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:29.632 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:29.633 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-01T02:43:29.633 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:29.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:29 smithi003 ceph-mon[67363]: pgmap v24: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:29.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:29.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:29.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-01T02:43:29.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:30.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:30 smithi003 ceph-mon[67363]: Reconfiguring osd.4 (monmap changed)... 2024-06-01T02:43:30.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:30 smithi003 ceph-mon[67363]: Reconfiguring daemon osd.4 on smithi027 2024-06-01T02:43:30.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:30 smithi027 ceph-mon[53801]: Reconfiguring osd.4 (monmap changed)... 2024-06-01T02:43:30.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:30 smithi027 ceph-mon[53801]: Reconfiguring daemon osd.4 on smithi027 2024-06-01T02:43:31.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:31 smithi003 ceph-mon[67363]: pgmap v25: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:31.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:31.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:31 smithi003 ceph-mon[67363]: Reconfiguring osd.5 (monmap changed)... 2024-06-01T02:43:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-01T02:43:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:31 smithi003 ceph-mon[67363]: Reconfiguring daemon osd.5 on smithi027 2024-06-01T02:43:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:43:31.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:31 smithi027 ceph-mon[53801]: pgmap v25: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:31.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:31.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:31.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:31 smithi027 ceph-mon[53801]: Reconfiguring osd.5 (monmap changed)... 2024-06-01T02:43:31.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-01T02:43:31.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:31.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:31 smithi027 ceph-mon[53801]: Reconfiguring daemon osd.5 on smithi027 2024-06-01T02:43:31.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:43:33.307 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:33 smithi027 ceph-mon[53801]: pgmap v26: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:33.308 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:33 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:33.308 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:33 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:33.308 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:33 smithi027 ceph-mon[53801]: Reconfiguring osd.6 (monmap changed)... 2024-06-01T02:43:33.308 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:33 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-01T02:43:33.308 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:33 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:33.308 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:33 smithi027 ceph-mon[53801]: Reconfiguring daemon osd.6 on smithi027 2024-06-01T02:43:33.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:33 smithi003 ceph-mon[67363]: pgmap v26: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:33.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:33 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:33.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:33 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:33.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:33 smithi003 ceph-mon[67363]: Reconfiguring osd.6 (monmap changed)... 2024-06-01T02:43:33.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:33 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-01T02:43:33.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:33 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:33.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:33 smithi003 ceph-mon[67363]: Reconfiguring daemon osd.6 on smithi027 2024-06-01T02:43:35.023 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:35.023 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:35.023 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:34 smithi027 ceph-mon[53801]: Reconfiguring osd.7 (monmap changed)... 2024-06-01T02:43:35.023 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-01T02:43:35.023 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:35.023 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:34 smithi027 ceph-mon[53801]: Reconfiguring daemon osd.7 on smithi027 2024-06-01T02:43:35.023 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:34 smithi027 ceph-mon[53801]: pgmap v27: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:35.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:35.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:35.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:34 smithi003 ceph-mon[67363]: Reconfiguring osd.7 (monmap changed)... 2024-06-01T02:43:35.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-01T02:43:35.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:35.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:34 smithi003 ceph-mon[67363]: Reconfiguring daemon osd.7 on smithi027 2024-06-01T02:43:35.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:34 smithi003 ceph-mon[67363]: pgmap v27: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:36 smithi027 ceph-mon[53801]: Reconfiguring mds.foofs.smithi027.xaykap (monmap changed)... 2024-06-01T02:43:36.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi027.xaykap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:43:36.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:36.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:36 smithi027 ceph-mon[53801]: Reconfiguring daemon mds.foofs.smithi027.xaykap on smithi027 2024-06-01T02:43:36.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:36.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:36.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:36 smithi003 ceph-mon[67363]: Reconfiguring mds.foofs.smithi027.xaykap (monmap changed)... 2024-06-01T02:43:36.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi027.xaykap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:43:36.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:36.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:36 smithi003 ceph-mon[67363]: Reconfiguring daemon mds.foofs.smithi027.xaykap on smithi027 2024-06-01T02:43:37.663 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: pgmap v28: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:37.663 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:37.663 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:37.663 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:43:37.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:37.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:37.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:37.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi003"}]: dispatch 2024-06-01T02:43:37.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi003"}]': finished 2024-06-01T02:43:37.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi027"}]: dispatch 2024-06-01T02:43:37.664 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi027"}]': finished 2024-06-01T02:43:37.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: pgmap v28: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:37.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:37.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:37.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:43:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi003"}]: dispatch 2024-06-01T02:43:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi003"}]': finished 2024-06-01T02:43:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi027"}]: dispatch 2024-06-01T02:43:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi027"}]': finished 2024-06-01T02:43:38.659 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:38 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all mon 2024-06-01T02:43:38.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:38 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all mon 2024-06-01T02:43:39.477 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:39 smithi003 ceph-mon[67363]: pgmap v29: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:39.477 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:39 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:39.477 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:39 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi003", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:43:39.477 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:39 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:39.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:39 smithi027 ceph-mon[53801]: pgmap v29: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:39.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:39 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:39.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:39 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi003", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:43:39.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:39 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:40.691 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:40 smithi003 ceph-mon[67363]: Upgrade: Updating crash.smithi003 (1/2) 2024-06-01T02:43:40.691 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:40 smithi003 ceph-mon[67363]: Deploying daemon crash.smithi003 on smithi003 2024-06-01T02:43:40.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:40 smithi027 ceph-mon[53801]: Upgrade: Updating crash.smithi003 (1/2) 2024-06-01T02:43:40.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:40 smithi027 ceph-mon[53801]: Deploying daemon crash.smithi003 on smithi003 2024-06-01T02:43:41.683 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:41 smithi003 ceph-mon[67363]: pgmap v30: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:41.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:41 smithi027 ceph-mon[53801]: pgmap v30: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:43.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:43 smithi003 ceph-mon[67363]: pgmap v31: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:43.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:43 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:43.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:43 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:43 smithi027 ceph-mon[53801]: pgmap v31: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:43 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:43 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:45.142 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:44 smithi027 ceph-mon[53801]: pgmap v32: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:45.142 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:44 smithi027 ceph-mon[53801]: Upgrade: Updating crash.smithi027 (2/2) 2024-06-01T02:43:45.142 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:44 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:45.143 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:44 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:43:45.143 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:44 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:45.143 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:44 smithi027 ceph-mon[53801]: Deploying daemon crash.smithi027 on smithi027 2024-06-01T02:43:45.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:44 smithi003 ceph-mon[67363]: pgmap v32: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:45.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:44 smithi003 ceph-mon[67363]: Upgrade: Updating crash.smithi027 (2/2) 2024-06-01T02:43:45.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:44 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:45.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:44 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi027", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-01T02:43:45.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:44 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:45.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:44 smithi003 ceph-mon[67363]: Deploying daemon crash.smithi027 on smithi027 2024-06-01T02:43:46.256 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:45 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:43:46.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:45 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:43:47.330 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:46 smithi027 ceph-mon[53801]: pgmap v33: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:47.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:46 smithi003 ceph-mon[67363]: pgmap v33: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:48.670 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:48 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:48.670 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:48 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:48.670 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:48 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:43:48.670 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:48 smithi027 ceph-mon[53801]: pgmap v34: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:48.682 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:48 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:48.682 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:48 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:48.682 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:48 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:43:48.682 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:48 smithi003 ceph-mon[67363]: pgmap v34: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:51.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:50 smithi003 ceph-mon[67363]: pgmap v35: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:51.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:50 smithi027 ceph-mon[53801]: pgmap v35: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:51.405 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:43:52.115 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:43:52.115 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (2m) 60s ago 25m 13.5M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:43:52.115 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 starting - - - - 2024-06-01T02:43:52.115 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (4s) 0s ago 25m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:43:52.115 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (2m) 60s ago 25m 44.1M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:43:52.115 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (20m) 60s ago 20m 40.6M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:43:52.115 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (20m) 0s ago 20m 18.2M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (2m) 60s ago 27m 476M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (90s) 0s ago 25m 451M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (75s) 60s ago 27m 29.7M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (51s) 0s ago 25m 33.9M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (19m) 60s ago 19m 278M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (19m) 0s ago 19m 88.2M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (2m) 60s ago 25m 8660k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (2m) 0s ago 25m 8418k - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (24m) 60s ago 24m 442M 1937M 17.2.0 e1d6a67b021e acddee4a7a8c 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (24m) 60s ago 24m 360M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (23m) 60s ago 23m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:43:52.116 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (23m) 60s ago 23m 370M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:43:52.117 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (22m) 0s ago 22m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:43:52.117 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (22m) 0s ago 22m 461M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:43:52.117 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (21m) 0s ago 21m 375M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:43:52.117 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (21m) 0s ago 21m 618M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:43:52.117 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (95s) 60s ago 24m 58.0M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:43:52.550 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:43:52.550 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:43:52.551 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-06-01T02:43:52.552 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 4 2024-06-01T02:43:52.552 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:43:52.552 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:43:52.552 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:52.552 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:52.552 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:52 smithi003 ceph-mon[67363]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:52 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:52 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:52 smithi027 ceph-mon[53801]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:52.926 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "5/23 daemons upgraded", 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading crash daemons", 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:43:52.927 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:43:53.345 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:43:53.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:53 smithi003 ceph-mon[67363]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:53.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:53 smithi003 ceph-mon[67363]: pgmap v36: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:53.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:53 smithi003 ceph-mon[67363]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:53.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:53 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:53.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:53 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:53.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:53 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1169168745' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:53.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:53 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/246095953' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:43:53.749 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:53 smithi027 ceph-mon[53801]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:53.749 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:53 smithi027 ceph-mon[53801]: pgmap v36: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:53.749 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:53 smithi027 ceph-mon[53801]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:53.749 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:53 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:53.749 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:53 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:53.749 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:53 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/1169168745' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:53.749 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:53 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/246095953' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:43:54.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:54 smithi003 ceph-mon[67363]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:54.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:54 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:54.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:54 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:54.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:54 smithi027 ceph-mon[53801]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:43:54.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:54 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:54.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:54 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:55.558 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:55 smithi003 ceph-mon[67363]: pgmap v37: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:55.558 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:55.558 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:55.764 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:55 smithi027 ceph-mon[53801]: pgmap v37: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:55.764 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:55.764 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:56.806 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:56 smithi003 ceph-mon[67363]: pgmap v38: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:56.806 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:56.806 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:57.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:56 smithi027 ceph-mon[53801]: pgmap v38: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:43:57.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:57.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: pgmap v39: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:59.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:59.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:43:59.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:43:59.240 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all crash 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi003"}]: dispatch 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi003"}]': finished 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi027"}]: dispatch 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi027"}]': finished 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-01T02:43:59.241 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:43:59 smithi003 ceph-mon[67363]: Upgrade: osd.0 is safe to restart 2024-06-01T02:43:59.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: pgmap v39: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:43:59.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all crash 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi003"}]: dispatch 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi003"}]': finished 2024-06-01T02:43:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi027"}]: dispatch 2024-06-01T02:43:59.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi027"}]': finished 2024-06-01T02:43:59.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-01T02:43:59.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-01T02:43:59.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:43:59 smithi027 ceph-mon[53801]: Upgrade: osd.0 is safe to restart 2024-06-01T02:44:01.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:01 smithi003 ceph-mon[67363]: pgmap v40: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:01 smithi003 ceph-mon[67363]: Upgrade: Updating osd.0 2024-06-01T02:44:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-01T02:44:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:44:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:01 smithi003 ceph-mon[67363]: Deploying daemon osd.0 on smithi003 2024-06-01T02:44:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:44:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:01 smithi027 ceph-mon[53801]: pgmap v40: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:01 smithi027 ceph-mon[53801]: Upgrade: Updating osd.0 2024-06-01T02:44:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-01T02:44:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:44:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:01 smithi027 ceph-mon[53801]: Deploying daemon osd.0 on smithi003 2024-06-01T02:44:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:44:03.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:03 smithi003 ceph-mon[67363]: pgmap v41: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:03.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:03 smithi027 ceph-mon[53801]: pgmap v41: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:03.614 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:03 smithi003 systemd[1]: Stopping Ceph osd.0 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:44:04.086 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:03 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:44:03.614+0000 7fdb9f1e2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:44:04.087 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:03 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:44:03.614+0000 7fdb9f1e2700 -1 osd.0 86 *** Got signal Terminated *** 2024-06-01T02:44:04.087 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:03 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[35054]: 2024-06-01T02:44:03.614+0000 7fdb9f1e2700 -1 osd.0 86 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:44:04.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:04 smithi003 ceph-mon[67363]: osd.0 marked itself down and dead 2024-06-01T02:44:04.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:04 smithi027 ceph-mon[53801]: osd.0 marked itself down and dead 2024-06-01T02:44:05.064 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:04 smithi003 podman[72990]: 2024-06-01 02:44:04.789759095 +0000 UTC m=+1.273606162 container died acddee4a7a8cec15d6e459ef82259731f46dcd88eb533f2a679be8eb487699c2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, version=8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, release=754, vcs-type=git) 2024-06-01T02:44:05.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:05 smithi003 ceph-mon[67363]: pgmap v42: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:05.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:05 smithi003 ceph-mon[67363]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:44:05.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:05 smithi003 ceph-mon[67363]: osdmap e87: 8 total, 7 up, 8 in 2024-06-01T02:44:05.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:05 smithi003 podman[72990]: 2024-06-01 02:44:05.138733298 +0000 UTC m=+1.622580357 container cleanup acddee4a7a8cec15d6e459ef82259731f46dcd88eb533f2a679be8eb487699c2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.buildah.version=1.19.8, release=754, version=8, RELEASE=HEAD, com.redhat.component=centos-stream-container, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-06-01T02:44:05.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:05 smithi003 bash[72990]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0 2024-06-01T02:44:05.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:05 smithi027 ceph-mon[53801]: pgmap v42: 97 pgs: 97 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:05.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:05 smithi027 ceph-mon[53801]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:44:05.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:05 smithi027 ceph-mon[53801]: osdmap e87: 8 total, 7 up, 8 in 2024-06-01T02:44:05.783 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:05 smithi003 podman[73007]: 2024-06-01 02:44:05.340271317 +0000 UTC m=+0.546078080 container remove acddee4a7a8cec15d6e459ef82259731f46dcd88eb533f2a679be8eb487699c2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, RELEASE=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.component=centos-stream-container, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870) 2024-06-01T02:44:06.086 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:05 smithi003 podman[73095]: 2024-06-01 02:44:05.686649105 +0000 UTC m=+0.018082701 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:44:06.087 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:05 smithi003 podman[73095]: 2024-06-01 02:44:05.791067532 +0000 UTC m=+0.122501101 container create c273bbfcbd836de57fe76f30250aee60aacfcf7da731e86e387057fddb7fe9a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, RELEASE=squid-3d89acc, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-06-01T02:44:06.421 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:06 smithi003 ceph-mon[67363]: osdmap e88: 8 total, 7 up, 8 in 2024-06-01T02:44:06.421 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:06 smithi003 podman[73095]: 2024-06-01 02:44:06.102478953 +0000 UTC m=+0.433912522 container init c273bbfcbd836de57fe76f30250aee60aacfcf7da731e86e387057fddb7fe9a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240528, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5) 2024-06-01T02:44:06.421 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:06 smithi003 podman[73095]: 2024-06-01 02:44:06.106007502 +0000 UTC m=+0.437441073 container start c273bbfcbd836de57fe76f30250aee60aacfcf7da731e86e387057fddb7fe9a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:44:06.421 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:06 smithi003 podman[73095]: 2024-06-01 02:44:06.210894405 +0000 UTC m=+0.542327990 container attach c273bbfcbd836de57fe76f30250aee60aacfcf7da731e86e387057fddb7fe9a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528) 2024-06-01T02:44:06.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:06 smithi027 ceph-mon[53801]: osdmap e88: 8 total, 7 up, 8 in 2024-06-01T02:44:06.706 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:06 smithi003 podman[73095]: 2024-06-01 02:44:06.421891205 +0000 UTC m=+0.753324785 container died c273bbfcbd836de57fe76f30250aee60aacfcf7da731e86e387057fddb7fe9a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-06-01T02:44:07.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:07 smithi003 ceph-mon[67363]: pgmap v45: 97 pgs: 13 stale+active+clean, 84 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:44:07.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:06 smithi003 podman[73120]: 2024-06-01 02:44:06.973069707 +0000 UTC m=+0.545104638 container remove c273bbfcbd836de57fe76f30250aee60aacfcf7da731e86e387057fddb7fe9a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc) 2024-06-01T02:44:07.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:06 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.0.service: Deactivated successfully. 2024-06-01T02:44:07.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:06 smithi003 systemd[1]: Stopped Ceph osd.0 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:44:07.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:06 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.0.service: Consumed 3min 41.280s CPU time. 2024-06-01T02:44:07.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:07 smithi003 systemd[1]: Starting Ceph osd.0 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:44:07.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:07 smithi027 ceph-mon[53801]: pgmap v45: 97 pgs: 13 stale+active+clean, 84 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:44:07.726 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:07 smithi003 podman[73218]: 2024-06-01 02:44:07.338397858 +0000 UTC m=+0.018179114 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:44:07.726 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:07 smithi003 podman[73218]: 2024-06-01 02:44:07.453698405 +0000 UTC m=+0.133479656 container create b4b5e461ddda9fdd119ef60912f38d6dc689bfd4bc93d3cd6351db98f2fceb5a (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-06-01T02:44:07.726 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:07 smithi003 podman[73218]: 2024-06-01 02:44:07.652432175 +0000 UTC m=+0.332213415 container init b4b5e461ddda9fdd119ef60912f38d6dc689bfd4bc93d3cd6351db98f2fceb5a (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-3d89acc, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:44:07.727 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:07 smithi003 podman[73218]: 2024-06-01 02:44:07.656373585 +0000 UTC m=+0.336154817 container start b4b5e461ddda9fdd119ef60912f38d6dc689bfd4bc93d3cd6351db98f2fceb5a (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-06-01T02:44:07.727 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:07 smithi003 podman[73218]: 2024-06-01 02:44:07.72699581 +0000 UTC m=+0.406777056 container attach b4b5e461ddda9fdd119ef60912f38d6dc689bfd4bc93d3cd6351db98f2fceb5a (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-06-01T02:44:08.457 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:08 smithi003 ceph-mon[67363]: Health check failed: Degraded data redundancy: 70/435 objects degraded (16.092%), 18 pgs degraded (PG_DEGRADED) 2024-06-01T02:44:08.457 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate[73232]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-01T02:44:08.457 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate[73232]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-01T02:44:08.457 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 bash[73218]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-01T02:44:08.457 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 bash[73218]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-01T02:44:08.457 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 bash[73218]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 bash[73218]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 bash[73218]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 bash[73218]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 bash[73218]: --> ceph-volume raw activate successful for osd ID: 0 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate[73232]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate[73232]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate[73232]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate[73232]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate[73232]: --> ceph-volume raw activate successful for osd ID: 0 2024-06-01T02:44:08.458 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 podman[73218]: 2024-06-01 02:44:08.150048529 +0000 UTC m=+0.829829773 container died b4b5e461ddda9fdd119ef60912f38d6dc689bfd4bc93d3cd6351db98f2fceb5a (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux ) 2024-06-01T02:44:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:08 smithi027 ceph-mon[53801]: Health check failed: Degraded data redundancy: 70/435 objects degraded (16.092%), 18 pgs degraded (PG_DEGRADED) 2024-06-01T02:44:09.066 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 podman[73353]: 2024-06-01 02:44:08.747345964 +0000 UTC m=+0.591623535 container remove b4b5e461ddda9fdd119ef60912f38d6dc689bfd4bc93d3cd6351db98f2fceb5a (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528) 2024-06-01T02:44:09.066 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 podman[73415]: 2024-06-01 02:44:08.964926347 +0000 UTC m=+0.111026981 container create cfa6fb225bc1a4102c6cc321e63d9d0aac8e87419a17a0f1131f697835e23672 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-06-01T02:44:09.066 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:08 smithi003 podman[73415]: 2024-06-01 02:44:08.872362402 +0000 UTC m=+0.018463071 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:44:09.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:09 smithi003 ceph-mon[67363]: pgmap v46: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:09.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:09 smithi003 podman[73415]: 2024-06-01 02:44:09.200675994 +0000 UTC m=+0.346776627 container init cfa6fb225bc1a4102c6cc321e63d9d0aac8e87419a17a0f1131f697835e23672 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-3d89acc) 2024-06-01T02:44:09.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:09 smithi003 podman[73415]: 2024-06-01 02:44:09.204017171 +0000 UTC m=+0.350117816 container start cfa6fb225bc1a4102c6cc321e63d9d0aac8e87419a17a0f1131f697835e23672 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528) 2024-06-01T02:44:09.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:09 smithi003 ceph-osd[73435]: -- 172.21.15.3:0/741475497 <== mon.0 v2:172.21.15.3:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ff333e9680 con 0x55ff333b2c00 2024-06-01T02:44:09.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:09 smithi003 bash[73415]: cfa6fb225bc1a4102c6cc321e63d9d0aac8e87419a17a0f1131f697835e23672 2024-06-01T02:44:09.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:09 smithi003 systemd[1]: Started Ceph osd.0 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:44:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:09 smithi027 ceph-mon[53801]: pgmap v46: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:10.086 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:09 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[73431]: 2024-06-01T02:44:09.749+0000 7f2803e847c0 -1 Falling back to public interface 2024-06-01T02:44:10.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:10 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:10.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:10 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:10.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:10 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:44:10.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:10 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:10.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:10 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:10.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:10 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:44:11.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:11 smithi027 ceph-mon[53801]: pgmap v47: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:11.768 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:11 smithi003 ceph-mon[67363]: pgmap v47: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:13.458 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:13 smithi003 ceph-mon[67363]: pgmap v48: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:13.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:13 smithi027 ceph-mon[53801]: pgmap v48: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:15.836 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:15 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[73431]: 2024-06-01T02:44:15.355+0000 7f2803e847c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-06-01T02:44:15.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:15 smithi003 ceph-mon[67363]: pgmap v49: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 213 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:15.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:15.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:15.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:15 smithi027 ceph-mon[53801]: pgmap v49: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 213 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:15.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:15.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:16.810 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:16 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[73431]: 2024-06-01T02:44:16.387+0000 7f2803e847c0 -1 osd.0 86 log_to_monitors true 2024-06-01T02:44:17.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:16 smithi003 ceph-mon[67363]: pgmap v50: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 193 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:17.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:17.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:44:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:16 smithi003 ceph-mon[67363]: from='osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-01T02:44:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:16 smithi027 ceph-mon[53801]: pgmap v50: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 193 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:44:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:16 smithi027 ceph-mon[53801]: from='osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-01T02:44:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:17.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:18.086 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:44:17 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[73431]: 2024-06-01T02:44:17.827+0000 7f27f12d3700 -1 osd.0 86 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:44:18.544 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:18 smithi027 ceph-mon[53801]: from='osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-01T02:44:18.544 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:18 smithi027 ceph-mon[53801]: osdmap e89: 8 total, 7 up, 8 in 2024-06-01T02:44:18.544 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:18 smithi027 ceph-mon[53801]: from='osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:44:18.544 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:18 smithi027 ceph-mon[53801]: from='osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212]' entity='osd.0' 2024-06-01T02:44:18.659 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:18 smithi003 ceph-mon[67363]: from='osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-01T02:44:18.659 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:18 smithi003 ceph-mon[67363]: osdmap e89: 8 total, 7 up, 8 in 2024-06-01T02:44:18.659 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:18 smithi003 ceph-mon[67363]: from='osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:44:18.659 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:18 smithi003 ceph-mon[67363]: from='osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212]' entity='osd.0' 2024-06-01T02:44:19.522 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:19 smithi003 ceph-mon[67363]: pgmap v52: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 307 B/s wr, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:19.522 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:19 smithi003 ceph-mon[67363]: mgrmap e41: smithi003.njsqxz(active, since 92s), standbys: smithi027.tujskj 2024-06-01T02:44:19.523 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:19 smithi003 ceph-mon[67363]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:44:19.523 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:19 smithi003 ceph-mon[67363]: osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212] boot 2024-06-01T02:44:19.523 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:19 smithi003 ceph-mon[67363]: osdmap e90: 8 total, 8 up, 8 in 2024-06-01T02:44:19.523 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:19 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:44:19.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:19 smithi027 ceph-mon[53801]: pgmap v52: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 307 B/s wr, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:19.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:19 smithi027 ceph-mon[53801]: mgrmap e41: smithi003.njsqxz(active, since 92s), standbys: smithi027.tujskj 2024-06-01T02:44:19.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:19 smithi027 ceph-mon[53801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:44:19.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:19 smithi027 ceph-mon[53801]: osd.0 [v2:172.21.15.3:6802/2265170212,v1:172.21.15.3:6803/2265170212] boot 2024-06-01T02:44:19.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:19 smithi027 ceph-mon[53801]: osdmap e90: 8 total, 8 up, 8 in 2024-06-01T02:44:19.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:19 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-01T02:44:21.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: pgmap v54: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 383 B/s wr, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: osdmap e91: 8 total, 8 up, 8 in 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:21.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:21.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-01T02:44:21.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-01T02:44:21.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:20 smithi003 ceph-mon[67363]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-06-01T02:44:21.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: pgmap v54: 97 pgs: 18 active+undersized+degraded, 18 active+undersized, 61 active+clean; 240 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 383 B/s wr, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: osdmap e91: 8 total, 8 up, 8 in 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-01T02:44:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-01T02:44:21.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:20 smithi027 ceph-mon[53801]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-06-01T02:44:22.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:21 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 13/435 objects degraded (2.989%), 2 pgs degraded (PG_DEGRADED) 2024-06-01T02:44:22.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:21 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 13/435 objects degraded (2.989%), 2 pgs degraded (PG_DEGRADED) 2024-06-01T02:44:23.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:23 smithi003 ceph-mon[67363]: pgmap v56: 97 pgs: 2 active+undersized+degraded, 11 active+undersized, 84 active+clean; 240 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 0 op/s; 13/435 objects degraded (2.989%) 2024-06-01T02:44:23.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:23 smithi027 ceph-mon[53801]: pgmap v56: 97 pgs: 2 active+undersized+degraded, 11 active+undersized, 84 active+clean; 240 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 511 B/s wr, 0 op/s; 13/435 objects degraded (2.989%) 2024-06-01T02:44:23.723 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:44:24.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:24 smithi027 ceph-mon[53801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/435 objects degraded (2.989%), 2 pgs degraded) 2024-06-01T02:44:24.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:24 smithi027 ceph-mon[53801]: Cluster is now healthy 2024-06-01T02:44:24.378 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:24 smithi003 ceph-mon[67363]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/435 objects degraded (2.989%), 2 pgs degraded) 2024-06-01T02:44:24.378 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:24 smithi003 ceph-mon[67363]: Cluster is now healthy 2024-06-01T02:44:24.406 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (2m) 9s ago 26m 14.1M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (42s) 9s ago 26m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (37s) 33s ago 25m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (2m) 9s ago 26m 43.9M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (20m) 9s ago 20m 40.5M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (20m) 33s ago 20m 18.2M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (2m) 9s ago 27m 503M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (2m) 33s ago 25m 451M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (107s) 9s ago 27m 44.8M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (83s) 33s ago 25m 33.9M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:44:24.407 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (20m) 9s ago 20m 292M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (20m) 33s ago 20m 88.2M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (2m) 9s ago 26m 10.6M - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (2m) 33s ago 25m 8418k - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (15s) 9s ago 25m 14.8M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (24m) 9s ago 24m 360M 1937M 17.2.0 e1d6a67b021e 49cb8109e651 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (24m) 9s ago 24m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (23m) 9s ago 23m 370M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (23m) 33s ago 23m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (22m) 33s ago 22m 461M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (22m) 33s ago 22m 375M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:44:24.408 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (22m) 33s ago 22m 618M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:44:24.409 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (2m) 9s ago 25m 64.9M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:44:24.821 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:44:24.821 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:44:24.821 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:44:24.821 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:44:24.821 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 1 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 5 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:44:24.822 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:44:25.174 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:44:25.174 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "7/23 daemons upgraded", 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading osd daemons", 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:44:25.175 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:44:25.176 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:25 smithi003 ceph-mon[67363]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:25.176 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:25 smithi003 ceph-mon[67363]: pgmap v57: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:44:25.176 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:25 smithi003 ceph-mon[67363]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:25.176 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:25 smithi003 ceph-mon[67363]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:25.176 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:25 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3743719198' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:25 smithi027 ceph-mon[53801]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:25 smithi027 ceph-mon[53801]: pgmap v57: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:44:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:25 smithi027 ceph-mon[53801]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:25.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:25 smithi027 ceph-mon[53801]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:25.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:25 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3743719198' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:25.593 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:44:26.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:26 smithi003 ceph-mon[67363]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:26.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:26 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1309125355' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:44:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:26 smithi027 ceph-mon[53801]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:26 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/1309125355' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:44:27.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:27 smithi003 ceph-mon[67363]: pgmap v58: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:44:27.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:27 smithi027 ceph-mon[53801]: pgmap v58: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-06-01T02:44:29.392 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:29 smithi027 ceph-mon[53801]: pgmap v59: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 229 B/s rd, 0 op/s 2024-06-01T02:44:29.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:29 smithi003 ceph-mon[67363]: pgmap v59: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 229 B/s rd, 0 op/s 2024-06-01T02:44:31.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:31 smithi003 ceph-mon[67363]: pgmap v60: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:44:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:44:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:31 smithi027 ceph-mon[53801]: pgmap v60: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:44:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:44:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:33.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:33 smithi003 ceph-mon[67363]: pgmap v61: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 171 B/s rd, 0 op/s 2024-06-01T02:44:33.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:33 smithi027 ceph-mon[53801]: pgmap v61: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 171 B/s rd, 0 op/s 2024-06-01T02:44:35.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:35 smithi003 ceph-mon[67363]: pgmap v62: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:35.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:35 smithi027 ceph-mon[53801]: pgmap v62: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:36.259 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-01T02:44:36.259 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:36 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-01T02:44:36.259 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:36 smithi003 ceph-mon[67363]: Upgrade: osd.1 is safe to restart 2024-06-01T02:44:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-01T02:44:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:36 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-01T02:44:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:36 smithi027 ceph-mon[53801]: Upgrade: osd.1 is safe to restart 2024-06-01T02:44:37.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:37 smithi003 ceph-mon[67363]: pgmap v63: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-01T02:44:37.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:44:37.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:37 smithi027 ceph-mon[53801]: pgmap v63: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:37.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:37.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-01T02:44:37.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:44:38.544 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:38 smithi003 ceph-mon[67363]: Upgrade: Updating osd.1 2024-06-01T02:44:38.544 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:38 smithi003 ceph-mon[67363]: Deploying daemon osd.1 on smithi003 2024-06-01T02:44:38.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:38 smithi027 ceph-mon[53801]: Upgrade: Updating osd.1 2024-06-01T02:44:38.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:38 smithi027 ceph-mon[53801]: Deploying daemon osd.1 on smithi003 2024-06-01T02:44:39.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:39 smithi003 ceph-mon[67363]: pgmap v64: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:44:39.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:39 smithi027 ceph-mon[53801]: pgmap v64: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:44:40.336 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:40 smithi003 systemd[1]: Stopping Ceph osd.1 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:44:40.836 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:40 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:44:40.405+0000 7f0a7db90700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:44:40.837 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:40 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:44:40.405+0000 7f0a7db90700 -1 osd.1 91 *** Got signal Terminated *** 2024-06-01T02:44:40.837 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:40 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[38255]: 2024-06-01T02:44:40.405+0000 7f0a7db90700 -1 osd.1 91 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:44:41.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:41 smithi003 ceph-mon[67363]: pgmap v65: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:41.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:41 smithi003 ceph-mon[67363]: osd.1 marked itself down and dead 2024-06-01T02:44:41.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:41 smithi027 ceph-mon[53801]: pgmap v65: 97 pgs: 97 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:44:41.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:41 smithi027 ceph-mon[53801]: osd.1 marked itself down and dead 2024-06-01T02:44:42.087 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:41 smithi003 podman[75610]: 2024-06-01 02:44:41.695985452 +0000 UTC m=+1.387547777 container died 49cb8109e6514a498e12cd9fc40690736caa429ee08092f60f81d7d47fa50245 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_CLEAN=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.buildah.version=1.19.8, release=754, com.redhat.component=centos-stream-container, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream) 2024-06-01T02:44:42.440 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:42 smithi003 ceph-mon[67363]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:44:42.440 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:42 smithi003 ceph-mon[67363]: osdmap e92: 8 total, 7 up, 8 in 2024-06-01T02:44:42.441 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:42 smithi003 podman[75610]: 2024-06-01 02:44:42.16472086 +0000 UTC m=+1.856283186 container cleanup 49cb8109e6514a498e12cd9fc40690736caa429ee08092f60f81d7d47fa50245 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1, distribution-scope=public, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, ceph=True, architecture=x86_64, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream) 2024-06-01T02:44:42.441 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:42 smithi003 bash[75610]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1 2024-06-01T02:44:42.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:42 smithi027 ceph-mon[53801]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:44:42.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:42 smithi027 ceph-mon[53801]: osdmap e92: 8 total, 7 up, 8 in 2024-06-01T02:44:42.795 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:42 smithi003 podman[75627]: 2024-06-01 02:44:42.441197643 +0000 UTC m=+0.736700801 container remove 49cb8109e6514a498e12cd9fc40690736caa429ee08092f60f81d7d47fa50245 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1, vcs-type=git, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., version=8, release=754, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, name=centos-stream, RELEASE=HEAD) 2024-06-01T02:44:43.049 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:42 smithi003 podman[75720]: 2024-06-01 02:44:42.700649546 +0000 UTC m=+0.020665502 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:44:43.049 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:42 smithi003 podman[75720]: 2024-06-01 02:44:42.800329162 +0000 UTC m=+0.120345105 container create 068ec05b213b7606a140faa929c32914e2fdf1b0bbe522d7171ac35e41a9e612 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-06-01T02:44:43.049 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:43 smithi003 podman[75720]: 2024-06-01 02:44:43.044651923 +0000 UTC m=+0.364667858 container init 068ec05b213b7606a140faa929c32914e2fdf1b0bbe522d7171ac35e41a9e612 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-3d89acc, ceph=True, org.label-schema.license=GPLv2) 2024-06-01T02:44:43.049 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:43 smithi003 podman[75720]: 2024-06-01 02:44:43.049318258 +0000 UTC m=+0.369334200 container start 068ec05b213b7606a140faa929c32914e2fdf1b0bbe522d7171ac35e41a9e612 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_CLEAN=True) 2024-06-01T02:44:43.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:43 smithi003 ceph-mon[67363]: pgmap v67: 97 pgs: 35 peering, 1 stale+active+clean, 61 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:44:43.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:43 smithi003 ceph-mon[67363]: Health check failed: Reduced data availability: 5 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-06-01T02:44:43.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:43 smithi003 ceph-mon[67363]: osdmap e93: 8 total, 7 up, 8 in 2024-06-01T02:44:43.337 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:43 smithi003 podman[75720]: 2024-06-01 02:44:43.144506148 +0000 UTC m=+0.464522093 container attach 068ec05b213b7606a140faa929c32914e2fdf1b0bbe522d7171ac35e41a9e612 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:44:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:43 smithi027 ceph-mon[53801]: pgmap v67: 97 pgs: 35 peering, 1 stale+active+clean, 61 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:44:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:43 smithi027 ceph-mon[53801]: Health check failed: Reduced data availability: 5 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-06-01T02:44:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:43 smithi027 ceph-mon[53801]: osdmap e93: 8 total, 7 up, 8 in 2024-06-01T02:44:43.638 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:43 smithi003 podman[75720]: 2024-06-01 02:44:43.350606919 +0000 UTC m=+0.670622861 container died 068ec05b213b7606a140faa929c32914e2fdf1b0bbe522d7171ac35e41a9e612 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-06-01T02:44:44.215 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:44 smithi003 ceph-mon[67363]: Health check failed: Degraded data redundancy: 5/435 objects degraded (1.149%), 1 pg degraded (PG_DEGRADED) 2024-06-01T02:44:44.215 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:43 smithi003 podman[75746]: 2024-06-01 02:44:43.965184581 +0000 UTC m=+0.608717781 container remove 068ec05b213b7606a140faa929c32914e2fdf1b0bbe522d7171ac35e41a9e612 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:44:44.216 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:43 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.1.service: Deactivated successfully. 2024-06-01T02:44:44.216 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:43 smithi003 systemd[1]: Stopped Ceph osd.1 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:44:44.216 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:43 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.1.service: Consumed 3min 42.963s CPU time. 2024-06-01T02:44:44.216 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:44 smithi003 systemd[1]: Starting Ceph osd.1 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:44:44.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:44 smithi027 ceph-mon[53801]: Health check failed: Degraded data redundancy: 5/435 objects degraded (1.149%), 1 pg degraded (PG_DEGRADED) 2024-06-01T02:44:44.643 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:44 smithi003 podman[75846]: 2024-06-01 02:44:44.424510974 +0000 UTC m=+0.104886637 container create 6de38bc6963b15fe289e018ee4c58d6e4256cd4efcbd214d8d3b245edddab5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate, RELEASE=squid-3d89acc, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:44:44.643 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:44 smithi003 podman[75846]: 2024-06-01 02:44:44.337763988 +0000 UTC m=+0.018139667 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:44:44.643 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:44 smithi003 podman[75846]: 2024-06-01 02:44:44.643675441 +0000 UTC m=+0.324051116 container init 6de38bc6963b15fe289e018ee4c58d6e4256cd4efcbd214d8d3b245edddab5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc) 2024-06-01T02:44:45.074 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:44 smithi003 podman[75846]: 2024-06-01 02:44:44.647307802 +0000 UTC m=+0.327683486 container start 6de38bc6963b15fe289e018ee4c58d6e4256cd4efcbd214d8d3b245edddab5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD) 2024-06-01T02:44:45.074 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:44 smithi003 podman[75846]: 2024-06-01 02:44:44.726819755 +0000 UTC m=+0.407195442 container attach 6de38bc6963b15fe289e018ee4c58d6e4256cd4efcbd214d8d3b245edddab5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-06-01T02:44:45.430 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:45 smithi003 ceph-mon[67363]: pgmap v69: 97 pgs: 1 active+undersized+degraded, 35 peering, 3 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 5/435 objects degraded (1.149%) 2024-06-01T02:44:45.430 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate[75862]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-01T02:44:45.430 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate[75862]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-01T02:44:45.430 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 bash[75846]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-01T02:44:45.430 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 bash[75846]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 bash[75846]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 bash[75846]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 bash[75846]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 bash[75846]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 bash[75846]: --> ceph-volume raw activate successful for osd ID: 1 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate[75862]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate[75862]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate[75862]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate[75862]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate[75862]: --> ceph-volume raw activate successful for osd ID: 1 2024-06-01T02:44:45.431 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 podman[75846]: 2024-06-01 02:44:45.137146585 +0000 UTC m=+0.817522266 container died 6de38bc6963b15fe289e018ee4c58d6e4256cd4efcbd214d8d3b245edddab5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:44:45.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:45 smithi027 ceph-mon[53801]: pgmap v69: 97 pgs: 1 active+undersized+degraded, 35 peering, 3 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 5/435 objects degraded (1.149%) 2024-06-01T02:44:46.027 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 podman[75998]: 2024-06-01 02:44:45.688716197 +0000 UTC m=+0.545945997 container remove 6de38bc6963b15fe289e018ee4c58d6e4256cd4efcbd214d8d3b245edddab5d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:44:46.027 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 podman[76057]: 2024-06-01 02:44:45.809723618 +0000 UTC m=+0.018103588 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:44:46.027 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:45 smithi003 podman[76057]: 2024-06-01 02:44:45.916959182 +0000 UTC m=+0.125339142 container create 2806eedd6c53f0c3e62dc00eaaf7febaf00998ed606ecd9b60ab50b8a6841415 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-06-01T02:44:46.586 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:46 smithi003 podman[76057]: 2024-06-01 02:44:46.160878363 +0000 UTC m=+0.369258327 container init 2806eedd6c53f0c3e62dc00eaaf7febaf00998ed606ecd9b60ab50b8a6841415 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:44:46.587 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:46 smithi003 podman[76057]: 2024-06-01 02:44:46.16422591 +0000 UTC m=+0.372605864 container start 2806eedd6c53f0c3e62dc00eaaf7febaf00998ed606ecd9b60ab50b8a6841415 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5) 2024-06-01T02:44:46.587 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:46 smithi003 ceph-osd[76076]: -- 172.21.15.3:0/1433211574 <== mon.0 v2:172.21.15.3:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a6579f1680 con 0x55a6579bac00 2024-06-01T02:44:46.587 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:46 smithi003 bash[76057]: 2806eedd6c53f0c3e62dc00eaaf7febaf00998ed606ecd9b60ab50b8a6841415 2024-06-01T02:44:46.587 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:46 smithi003 systemd[1]: Started Ceph osd.1 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:44:47.086 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:46 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:44:46.706+0000 7f63cc0d67c0 -1 Falling back to public interface 2024-06-01T02:44:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:46 smithi003 ceph-mon[67363]: pgmap v70: 97 pgs: 1 active+undersized+degraded, 35 peering, 3 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5/435 objects degraded (1.149%) 2024-06-01T02:44:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:44:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:44:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:46 smithi027 ceph-mon[53801]: pgmap v70: 97 pgs: 1 active+undersized+degraded, 35 peering, 3 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5/435 objects degraded (1.149%) 2024-06-01T02:44:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:44:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:44:48.208 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:47 smithi003 ceph-mon[67363]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 12 pgs peering) 2024-06-01T02:44:48.307 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:47 smithi027 ceph-mon[53801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 12 pgs peering) 2024-06-01T02:44:48.962 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:48 smithi003 ceph-mon[67363]: pgmap v71: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-06-01T02:44:49.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:48 smithi027 ceph-mon[53801]: pgmap v71: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-06-01T02:44:51.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:50 smithi003 ceph-mon[67363]: pgmap v72: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 237 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-06-01T02:44:51.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:50 smithi027 ceph-mon[53801]: pgmap v72: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 237 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-06-01T02:44:52.086 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:51 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:44:51.761+0000 7f63cc0d67c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-06-01T02:44:53.086 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:52 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:44:52.642+0000 7f63cc0d67c0 -1 osd.1 91 log_to_monitors true 2024-06-01T02:44:53.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:52 smithi003 ceph-mon[67363]: pgmap v73: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-06-01T02:44:53.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:52 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 48/435 objects degraded (11.034%), 13 pgs degraded (PG_DEGRADED) 2024-06-01T02:44:53.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:53.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:53.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:52 smithi003 ceph-mon[67363]: from='osd.1 [v2:172.21.15.3:6810/838195919,v1:172.21.15.3:6811/838195919]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-01T02:44:53.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:52 smithi027 ceph-mon[53801]: pgmap v73: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-06-01T02:44:53.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:52 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 48/435 objects degraded (11.034%), 13 pgs degraded (PG_DEGRADED) 2024-06-01T02:44:53.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:52 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:53.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:52 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:53.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:52 smithi027 ceph-mon[53801]: from='osd.1 [v2:172.21.15.3:6810/838195919,v1:172.21.15.3:6811/838195919]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-01T02:44:53.504 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:44:53 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:44:53.145+0000 7f63bd52d700 -1 osd.1 91 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:44:54.247 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:54 smithi003 ceph-mon[67363]: from='osd.1 [v2:172.21.15.3:6810/838195919,v1:172.21.15.3:6811/838195919]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-01T02:44:54.247 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:54 smithi003 ceph-mon[67363]: osdmap e94: 8 total, 7 up, 8 in 2024-06-01T02:44:54.247 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:54 smithi003 ceph-mon[67363]: from='osd.1 [v2:172.21.15.3:6810/838195919,v1:172.21.15.3:6811/838195919]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:44:54.247 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:54 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:54.247 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:54 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:54 smithi027 ceph-mon[53801]: from='osd.1 [v2:172.21.15.3:6810/838195919,v1:172.21.15.3:6811/838195919]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-01T02:44:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:54 smithi027 ceph-mon[53801]: osdmap e94: 8 total, 7 up, 8 in 2024-06-01T02:44:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:54 smithi027 ceph-mon[53801]: from='osd.1 [v2:172.21.15.3:6810/838195919,v1:172.21.15.3:6811/838195919]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:44:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:54 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:54 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:55.199 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:55 smithi003 ceph-mon[67363]: pgmap v75: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-06-01T02:44:55.199 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:55 smithi003 ceph-mon[67363]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:44:55.199 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:55 smithi003 ceph-mon[67363]: osd.1 [v2:172.21.15.3:6810/838195919,v1:172.21.15.3:6811/838195919] boot 2024-06-01T02:44:55.199 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:55 smithi003 ceph-mon[67363]: osdmap e95: 8 total, 8 up, 8 in 2024-06-01T02:44:55.199 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:44:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:55 smithi027 ceph-mon[53801]: pgmap v75: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-06-01T02:44:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:55 smithi027 ceph-mon[53801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:44:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:55 smithi027 ceph-mon[53801]: osd.1 [v2:172.21.15.3:6810/838195919,v1:172.21.15.3:6811/838195919] boot 2024-06-01T02:44:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:55 smithi027 ceph-mon[53801]: osdmap e95: 8 total, 8 up, 8 in 2024-06-01T02:44:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-01T02:44:55.968 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:44:56.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:56 smithi003 ceph-mon[67363]: osdmap e96: 8 total, 8 up, 8 in 2024-06-01T02:44:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:56 smithi027 ceph-mon[53801]: osdmap e96: 8 total, 8 up, 8 in 2024-06-01T02:44:56.697 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:44:56.697 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (3m) 4s ago 27m 14.1M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:44:56.697 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (74s) 4s ago 26m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:44:56.697 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (69s) 65s ago 26m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (3m) 4s ago 26m 44.6M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (21m) 4s ago 21m 40.6M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (21m) 65s ago 21m 18.2M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (3m) 4s ago 28m 505M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (2m) 65s ago 26m 451M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (2m) 4s ago 28m 46.6M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (116s) 65s ago 26m 33.9M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (20m) 4s ago 20m 294M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (20m) 65s ago 20m 88.2M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (3m) 4s ago 26m 9252k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (3m) 65s ago 26m 8418k - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (47s) 4s ago 25m 191M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:44:56.698 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (10s) 4s ago 25m 14.9M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:44:56.699 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (24m) 4s ago 24m 490M 1937M 17.2.0 e1d6a67b021e e024b480c31a 2024-06-01T02:44:56.699 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (24m) 4s ago 24m 369M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:44:56.699 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (23m) 65s ago 23m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:44:56.699 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (23m) 65s ago 23m 461M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:44:56.699 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (23m) 65s ago 23m 375M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:44:56.699 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (22m) 65s ago 22m 618M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:44:56.699 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (2m) 4s ago 25m 67.4M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:44:57.120 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:44:57.121 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:44:57.121 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:44:57.121 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:44:57.121 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-06-01T02:44:57.121 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 6 2024-06-01T02:44:57.121 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:44:57.121 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:44:57.466 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:44:57.466 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:44:57.466 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:44:57.466 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:44:57.466 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:44:57.466 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:44:57.466 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:44:57.467 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:44:57.467 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:44:57.467 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "8/23 daemons upgraded", 2024-06-01T02:44:57.467 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading osd daemons", 2024-06-01T02:44:57.467 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:44:57.467 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:44:57.467 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: pgmap v78: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:44:57.467 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:57.467 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='client.34240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:57.467 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:57.467 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-01T02:44:57.468 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:57 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3261908567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: pgmap v78: 97 pgs: 13 active+undersized+degraded, 26 active+undersized, 58 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:44:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='client.34240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-01T02:44:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:57 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3261908567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:44:57.881 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_WARN Degraded data redundancy: 48/435 objects degraded (11.034%), 13 pgs degraded 2024-06-01T02:44:57.881 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 48/435 objects degraded (11.034%), 13 pgs degraded 2024-06-01T02:44:57.881 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-06-01T02:44:57.881 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-06-01T02:44:57.881 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-06-01T02:44:57.881 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-06-01T02:44:57.881 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-06-01T02:44:57.881 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-06-01T02:44:57.881 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-06-01T02:44:57.882 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-06-01T02:44:57.882 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-06-01T02:44:57.882 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-06-01T02:44:57.882 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-06-01T02:44:57.882 INFO:teuthology.orchestra.run.smithi003.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-06-01T02:44:57.882 INFO:teuthology.orchestra.run.smithi003.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-06-01T02:44:58.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:58 smithi003 ceph-mon[67363]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:58.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:58 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-01T02:44:58.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:58 smithi003 ceph-mon[67363]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-06-01T02:44:58.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:58 smithi003 ceph-mon[67363]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:58.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:58 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1986622707' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:44:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:58 smithi027 ceph-mon[53801]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:58 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-01T02:44:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:58 smithi027 ceph-mon[53801]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-06-01T02:44:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:58 smithi027 ceph-mon[53801]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:44:58.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:58 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/1986622707' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:44:59.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:59 smithi003 ceph-mon[67363]: pgmap v79: 97 pgs: 6 active+undersized+degraded, 7 active+undersized, 84 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 27/435 objects degraded (6.207%) 2024-06-01T02:44:59.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:44:59 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 27/435 objects degraded (6.207%), 6 pgs degraded (PG_DEGRADED) 2024-06-01T02:44:59.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:59 smithi027 ceph-mon[53801]: pgmap v79: 97 pgs: 6 active+undersized+degraded, 7 active+undersized, 84 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 27/435 objects degraded (6.207%) 2024-06-01T02:44:59.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:44:59 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 27/435 objects degraded (6.207%), 6 pgs degraded (PG_DEGRADED) 2024-06-01T02:45:00.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:00 smithi003 ceph-mon[67363]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/435 objects degraded (6.207%), 6 pgs degraded) 2024-06-01T02:45:00.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:00 smithi003 ceph-mon[67363]: Cluster is now healthy 2024-06-01T02:45:00.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:00 smithi027 ceph-mon[53801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/435 objects degraded (6.207%), 6 pgs degraded) 2024-06-01T02:45:00.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:00 smithi027 ceph-mon[53801]: Cluster is now healthy 2024-06-01T02:45:01.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:01 smithi003 ceph-mon[67363]: pgmap v80: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 308 B/s rd, 0 op/s 2024-06-01T02:45:01.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:01.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:45:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:01 smithi027 ceph-mon[53801]: pgmap v80: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 308 B/s rd, 0 op/s 2024-06-01T02:45:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:45:03.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:03 smithi003 ceph-mon[67363]: pgmap v81: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:45:03.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:03 smithi027 ceph-mon[53801]: pgmap v81: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:45:05.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:05 smithi003 ceph-mon[67363]: pgmap v82: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 212 B/s rd, 0 op/s 2024-06-01T02:45:05.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:05 smithi027 ceph-mon[53801]: pgmap v82: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 212 B/s rd, 0 op/s 2024-06-01T02:45:07.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:06 smithi027 ceph-mon[53801]: pgmap v83: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 192 B/s rd, 0 op/s 2024-06-01T02:45:07.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:06 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:07.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:06 smithi003 ceph-mon[67363]: pgmap v83: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 192 B/s rd, 0 op/s 2024-06-01T02:45:07.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:06 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:09.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:08 smithi003 ceph-mon[67363]: pgmap v84: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:45:09.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:08 smithi027 ceph-mon[53801]: pgmap v84: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:45:11.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:10 smithi027 ceph-mon[53801]: pgmap v85: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:11.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:10 smithi003 ceph-mon[67363]: pgmap v85: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:13.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:12 smithi003 ceph-mon[67363]: pgmap v86: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:13.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-01T02:45:13.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:12 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-01T02:45:13.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:12 smithi003 ceph-mon[67363]: Upgrade: osd.2 is safe to restart 2024-06-01T02:45:13.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:12 smithi027 ceph-mon[53801]: pgmap v86: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:13.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-01T02:45:13.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:12 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-01T02:45:13.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:12 smithi027 ceph-mon[53801]: Upgrade: osd.2 is safe to restart 2024-06-01T02:45:14.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:14 smithi003 ceph-mon[67363]: Upgrade: Updating osd.2 2024-06-01T02:45:14.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:14 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:14.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:14 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-01T02:45:14.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:14 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:45:14.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:14 smithi003 ceph-mon[67363]: Deploying daemon osd.2 on smithi003 2024-06-01T02:45:14.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:14 smithi003 ceph-mon[67363]: pgmap v87: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:14 smithi027 ceph-mon[53801]: Upgrade: Updating osd.2 2024-06-01T02:45:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:14 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:15.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:14 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-01T02:45:15.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:14 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:45:15.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:14 smithi027 ceph-mon[53801]: Deploying daemon osd.2 on smithi003 2024-06-01T02:45:15.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:14 smithi027 ceph-mon[53801]: pgmap v87: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:16.235 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:45:16.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:45:17.157 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:16 smithi003 ceph-mon[67363]: pgmap v88: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:17.157 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:16 smithi003 systemd[1]: Stopping Ceph osd.2 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:45:17.253 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:16 smithi027 ceph-mon[53801]: pgmap v88: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:17.586 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:17 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[42096]: 2024-06-01T02:45:17.156+0000 7ff4dbc21700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:45:17.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:17 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[42096]: 2024-06-01T02:45:17.156+0000 7ff4dbc21700 -1 osd.2 96 *** Got signal Terminated *** 2024-06-01T02:45:17.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:17 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[42096]: 2024-06-01T02:45:17.156+0000 7ff4dbc21700 -1 osd.2 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:45:18.330 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:17 smithi003 ceph-mon[67363]: osd.2 marked itself down and dead 2024-06-01T02:45:18.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:17 smithi027 ceph-mon[53801]: osd.2 marked itself down and dead 2024-06-01T02:45:18.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:18 smithi003 podman[78245]: 2024-06-01 02:45:18.331267721 +0000 UTC m=+1.272224344 container died e024b480c31a985f45eb4f304f0e09a88ee2e0cba9e91298416d3012482bf5ec (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, com.redhat.component=centos-stream-container, RELEASE=HEAD, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream) 2024-06-01T02:45:19.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:18 smithi003 ceph-mon[67363]: pgmap v89: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:45:19.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:18 smithi003 ceph-mon[67363]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:45:19.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:18 smithi003 ceph-mon[67363]: osdmap e97: 8 total, 7 up, 8 in 2024-06-01T02:45:19.087 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:18 smithi003 podman[78245]: 2024-06-01 02:45:18.648600204 +0000 UTC m=+1.589556825 container cleanup e024b480c31a985f45eb4f304f0e09a88ee2e0cba9e91298416d3012482bf5ec (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, vcs-type=git, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, distribution-scope=public) 2024-06-01T02:45:19.087 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:18 smithi003 bash[78245]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2 2024-06-01T02:45:19.087 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:18 smithi003 podman[78262]: 2024-06-01 02:45:18.826468135 +0000 UTC m=+0.495102928 container remove e024b480c31a985f45eb4f304f0e09a88ee2e0cba9e91298416d3012482bf5ec (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2, version=8, RELEASE=HEAD, distribution-scope=public, vcs-type=git, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vendor=Red Hat, Inc.) 2024-06-01T02:45:19.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:18 smithi027 ceph-mon[53801]: pgmap v89: 97 pgs: 97 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:45:19.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:18 smithi027 ceph-mon[53801]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:45:19.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:18 smithi027 ceph-mon[53801]: osdmap e97: 8 total, 7 up, 8 in 2024-06-01T02:45:19.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:19 smithi003 podman[78351]: 2024-06-01 02:45:19.292537532 +0000 UTC m=+0.106569562 container create 314a27ddd2cede88c85afbf6a6b95ab05522d077955b8a6660738f0f5657c3a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-06-01T02:45:19.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:19 smithi003 podman[78351]: 2024-06-01 02:45:19.204526012 +0000 UTC m=+0.018558041 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:45:19.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:19 smithi003 podman[78351]: 2024-06-01 02:45:19.51164678 +0000 UTC m=+0.325678781 container init 314a27ddd2cede88c85afbf6a6b95ab05522d077955b8a6660738f0f5657c3a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, RELEASE=squid-3d89acc, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-06-01T02:45:19.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:19 smithi003 podman[78351]: 2024-06-01 02:45:19.51519801 +0000 UTC m=+0.329230031 container start 314a27ddd2cede88c85afbf6a6b95ab05522d077955b8a6660738f0f5657c3a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528) 2024-06-01T02:45:19.876 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:19 smithi003 podman[78351]: 2024-06-01 02:45:19.594663449 +0000 UTC m=+0.408695469 container attach 314a27ddd2cede88c85afbf6a6b95ab05522d077955b8a6660738f0f5657c3a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:45:19.876 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:19 smithi003 podman[78351]: 2024-06-01 02:45:19.827690931 +0000 UTC m=+0.641722942 container died 314a27ddd2cede88c85afbf6a6b95ab05522d077955b8a6660738f0f5657c3a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:45:20.173 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:19 smithi003 ceph-mon[67363]: osdmap e98: 8 total, 7 up, 8 in 2024-06-01T02:45:20.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:19 smithi027 ceph-mon[53801]: osdmap e98: 8 total, 7 up, 8 in 2024-06-01T02:45:20.690 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:20 smithi003 podman[78376]: 2024-06-01 02:45:20.44004631 +0000 UTC m=+0.602973302 container remove 314a27ddd2cede88c85afbf6a6b95ab05522d077955b8a6660738f0f5657c3a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-3d89acc, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:45:20.691 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:20 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.2.service: Deactivated successfully. 2024-06-01T02:45:20.691 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:20 smithi003 systemd[1]: Stopped Ceph osd.2 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:45:20.691 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:20 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.2.service: Consumed 2min 29.634s CPU time. 2024-06-01T02:45:20.691 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:20 smithi003 systemd[1]: Starting Ceph osd.2 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:45:21.160 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:20 smithi003 podman[78476]: 2024-06-01 02:45:20.813942216 +0000 UTC m=+0.017942496 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:45:21.160 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:20 smithi003 podman[78476]: 2024-06-01 02:45:20.924457374 +0000 UTC m=+0.128457676 container create 4abc9de0dc3d0b396b371b55bac88dcfe47b18e03f31de98fa575d9c22e9b4d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-06-01T02:45:21.160 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:20 smithi003 ceph-mon[67363]: pgmap v92: 97 pgs: 5 peering, 5 stale+active+clean, 87 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:45:21.160 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:20 smithi003 ceph-mon[67363]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-01T02:45:21.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:20 smithi027 ceph-mon[53801]: pgmap v92: 97 pgs: 5 peering, 5 stale+active+clean, 87 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:45:21.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:20 smithi027 ceph-mon[53801]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-01T02:45:21.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 podman[78476]: 2024-06-01 02:45:21.160242556 +0000 UTC m=+0.364242825 container init 4abc9de0dc3d0b396b371b55bac88dcfe47b18e03f31de98fa575d9c22e9b4d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-06-01T02:45:21.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 podman[78476]: 2024-06-01 02:45:21.16424813 +0000 UTC m=+0.368248398 container start 4abc9de0dc3d0b396b371b55bac88dcfe47b18e03f31de98fa575d9c22e9b4d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:45:21.587 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 podman[78476]: 2024-06-01 02:45:21.251720334 +0000 UTC m=+0.455720606 container attach 4abc9de0dc3d0b396b371b55bac88dcfe47b18e03f31de98fa575d9c22e9b4d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240528, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-06-01T02:45:21.929 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate[78492]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-01T02:45:21.929 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate[78492]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-01T02:45:21.929 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 bash[78476]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-01T02:45:21.929 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 bash[78476]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 bash[78476]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 bash[78476]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 bash[78476]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 bash[78476]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 bash[78476]: --> ceph-volume raw activate successful for osd ID: 2 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate[78492]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate[78492]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate[78492]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate[78492]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate[78492]: --> ceph-volume raw activate successful for osd ID: 2 2024-06-01T02:45:21.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:21 smithi003 podman[78476]: 2024-06-01 02:45:21.661188617 +0000 UTC m=+0.865188879 container died 4abc9de0dc3d0b396b371b55bac88dcfe47b18e03f31de98fa575d9c22e9b4d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528) 2024-06-01T02:45:22.264 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:21 smithi003 ceph-mon[67363]: Health check failed: Degraded data redundancy: 42/435 objects degraded (9.655%), 10 pgs degraded (PG_DEGRADED) 2024-06-01T02:45:22.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:21 smithi027 ceph-mon[53801]: Health check failed: Degraded data redundancy: 42/435 objects degraded (9.655%), 10 pgs degraded (PG_DEGRADED) 2024-06-01T02:45:22.569 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:22 smithi003 podman[78629]: 2024-06-01 02:45:22.264107883 +0000 UTC m=+0.591410216 container remove 4abc9de0dc3d0b396b371b55bac88dcfe47b18e03f31de98fa575d9c22e9b4d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-06-01T02:45:22.569 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:22 smithi003 podman[78689]: 2024-06-01 02:45:22.481634231 +0000 UTC m=+0.114949053 container create bc04c16a878399eb5d45019099021d9e94840cc5e2f8936d3280fd7226d05eab (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-06-01T02:45:22.569 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:22 smithi003 podman[78689]: 2024-06-01 02:45:22.385112824 +0000 UTC m=+0.018427604 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:45:22.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:22 smithi003 podman[78689]: 2024-06-01 02:45:22.667101927 +0000 UTC m=+0.300416702 container init bc04c16a878399eb5d45019099021d9e94840cc5e2f8936d3280fd7226d05eab (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc) 2024-06-01T02:45:22.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:22 smithi003 podman[78689]: 2024-06-01 02:45:22.670534016 +0000 UTC m=+0.303848789 container start bc04c16a878399eb5d45019099021d9e94840cc5e2f8936d3280fd7226d05eab (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-06-01T02:45:22.930 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:22 smithi003 ceph-osd[78707]: -- 172.21.15.3:0/2127221926 <== mon.0 v2:172.21.15.3:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562431b23680 con 0x562431aecc00 2024-06-01T02:45:22.931 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:22 smithi003 bash[78689]: bc04c16a878399eb5d45019099021d9e94840cc5e2f8936d3280fd7226d05eab 2024-06-01T02:45:22.931 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:22 smithi003 systemd[1]: Started Ceph osd.2 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:45:23.219 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:22 smithi003 ceph-mon[67363]: pgmap v93: 97 pgs: 10 active+undersized+degraded, 5 peering, 9 active+undersized, 73 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 42/435 objects degraded (9.655%) 2024-06-01T02:45:23.219 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:23.219 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:23.219 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:45:23.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:22 smithi027 ceph-mon[53801]: pgmap v93: 97 pgs: 10 active+undersized+degraded, 5 peering, 9 active+undersized, 73 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 42/435 objects degraded (9.655%) 2024-06-01T02:45:23.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:23.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:23.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:45:23.504 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:23 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[78703]: 2024-06-01T02:45:23.219+0000 7ff7581b27c0 -1 Falling back to public interface 2024-06-01T02:45:24.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:23 smithi003 ceph-mon[67363]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-01T02:45:24.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:23 smithi027 ceph-mon[53801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-01T02:45:25.309 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:24 smithi003 ceph-mon[67363]: pgmap v94: 97 pgs: 11 active+undersized+degraded, 15 active+undersized, 71 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:45:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:24 smithi027 ceph-mon[53801]: pgmap v94: 97 pgs: 11 active+undersized+degraded, 15 active+undersized, 71 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:45:27.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:26 smithi003 ceph-mon[67363]: pgmap v95: 97 pgs: 11 active+undersized+degraded, 15 active+undersized, 71 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:45:27.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:26 smithi027 ceph-mon[53801]: pgmap v95: 97 pgs: 11 active+undersized+degraded, 15 active+undersized, 71 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:45:27.993 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:27 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[78703]: 2024-06-01T02:45:27.891+0000 7ff7581b27c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-06-01T02:45:28.276 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:45:28.277 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:27 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 48/435 objects degraded (11.034%), 11 pgs degraded (PG_DEGRADED) 2024-06-01T02:45:28.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:27 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 48/435 objects degraded (11.034%), 11 pgs degraded (PG_DEGRADED) 2024-06-01T02:45:28.836 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:28 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[78703]: 2024-06-01T02:45:28.677+0000 7ff7581b27c0 -1 osd.2 96 log_to_monitors true 2024-06-01T02:45:29.019 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (4m) 0s ago 27m 14.1M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (107s) 0s ago 27m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (101s) 97s ago 26m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (3m) 0s ago 27m 44.0M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (21m) 0s ago 21m 40.7M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (21m) 97s ago 21m 18.2M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (3m) 0s ago 28m 507M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (3m) 97s ago 26m 451M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (2m) 0s ago 28m 47.1M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (2m) 97s ago 26m 33.9M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (21m) 0s ago 21m 294M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (21m) 97s ago 21m 88.2M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:45:29.020 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (3m) 0s ago 27m 8862k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (3m) 97s ago 26m 8418k - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (80s) 0s ago 26m 192M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (43s) 0s ago 25m 174M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (6s) 0s ago 25m 11.5M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (24m) 0s ago 24m 370M 1937M 17.2.0 e1d6a67b021e e06e99c63f47 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (24m) 97s ago 24m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (23m) 97s ago 23m 461M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (23m) 97s ago 23m 375M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (23m) 97s ago 23m 618M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:45:29.021 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (3m) 0s ago 26m 67.4M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:45:29.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:28 smithi003 ceph-mon[67363]: pgmap v96: 97 pgs: 11 active+undersized+degraded, 15 active+undersized, 71 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:45:29.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:28 smithi003 ceph-mon[67363]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:45:29.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:28 smithi003 ceph-mon[67363]: from='osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-01T02:45:29.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:29.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:29.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:28 smithi027 ceph-mon[53801]: pgmap v96: 97 pgs: 11 active+undersized+degraded, 15 active+undersized, 71 active+clean; 240 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:45:29.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:28 smithi027 ceph-mon[53801]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:45:29.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:28 smithi027 ceph-mon[53801]: from='osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-01T02:45:29.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:29.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:29.492 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:45:29.492 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:45:29.492 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:45:29.492 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:45:29.492 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 6 2024-06-01T02:45:29.493 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:45:29.494 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:45:29.836 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:45:29 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[78703]: 2024-06-01T02:45:29.497+0000 7ff745601700 -1 osd.2 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:45:29.871 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:45:29.871 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:45:29.871 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:45:29.871 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:45:29.871 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:45:29.871 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:45:29.871 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:45:29.871 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:45:29.871 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:45:29.872 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "9/23 daemons upgraded", 2024-06-01T02:45:29.872 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading osd daemons", 2024-06-01T02:45:29.872 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:45:29.872 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:45:30.292 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 48/435 objects degraded (11.034%), 11 pgs degraded 2024-06-01T02:45:30.292 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] OSD_DOWN: 1 osds down 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: osd.2 (root=default,host=smithi003) is down 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 48/435 objects degraded (11.034%), 11 pgs degraded 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-06-01T02:45:30.293 INFO:teuthology.orchestra.run.smithi003.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-06-01T02:45:30.294 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:30 smithi003 ceph-mon[67363]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:45:30.294 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:30 smithi003 ceph-mon[67363]: from='osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-01T02:45:30.294 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:30 smithi003 ceph-mon[67363]: osdmap e99: 8 total, 7 up, 8 in 2024-06-01T02:45:30.294 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:30 smithi003 ceph-mon[67363]: from='osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:45:30.294 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:30 smithi003 ceph-mon[67363]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:45:30.294 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:30 smithi003 ceph-mon[67363]: from='osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053]' entity='osd.2' 2024-06-01T02:45:30.294 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:30 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3492153977' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:45:30.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:30 smithi027 ceph-mon[53801]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:45:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:30 smithi027 ceph-mon[53801]: from='osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-01T02:45:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:30 smithi027 ceph-mon[53801]: osdmap e99: 8 total, 7 up, 8 in 2024-06-01T02:45:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:30 smithi027 ceph-mon[53801]: from='osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:45:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:30 smithi027 ceph-mon[53801]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:45:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:30 smithi027 ceph-mon[53801]: from='osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053]' entity='osd.2' 2024-06-01T02:45:30.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:30 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3492153977' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:45:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: pgmap v98: 97 pgs: 11 active+undersized+degraded, 15 active+undersized, 71 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:45:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:45:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3472323518' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:45:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:45:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053] boot 2024-06-01T02:45:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: osdmap e100: 8 total, 8 up, 8 in 2024-06-01T02:45:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:45:31.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:31.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:45:31.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: pgmap v98: 97 pgs: 11 active+undersized+degraded, 15 active+undersized, 71 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:45:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:45:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3472323518' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:45:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:45:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: osd.2 [v2:172.21.15.3:6818/3143724053,v1:172.21.15.3:6819/3143724053] boot 2024-06-01T02:45:31.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: osdmap e100: 8 total, 8 up, 8 in 2024-06-01T02:45:31.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-01T02:45:31.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:31.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:45:31.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:32.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:32 smithi003 ceph-mon[67363]: osdmap e101: 8 total, 8 up, 8 in 2024-06-01T02:45:32.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:32 smithi027 ceph-mon[53801]: osdmap e101: 8 total, 8 up, 8 in 2024-06-01T02:45:33.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:33 smithi003 ceph-mon[67363]: pgmap v101: 97 pgs: 4 active+undersized+degraded, 8 active+undersized, 85 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 25/435 objects degraded (5.747%) 2024-06-01T02:45:33.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:33 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 25/435 objects degraded (5.747%), 4 pgs degraded (PG_DEGRADED) 2024-06-01T02:45:33.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:33 smithi027 ceph-mon[53801]: pgmap v101: 97 pgs: 4 active+undersized+degraded, 8 active+undersized, 85 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 25/435 objects degraded (5.747%) 2024-06-01T02:45:33.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:33 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 25/435 objects degraded (5.747%), 4 pgs degraded (PG_DEGRADED) 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: pgmap v102: 97 pgs: 4 active+undersized+degraded, 2 active+undersized, 91 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 25/435 objects degraded (5.747%) 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-01T02:45:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-01T02:45:35.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:34 smithi003 ceph-mon[67363]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-06-01T02:45:35.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: pgmap v102: 97 pgs: 4 active+undersized+degraded, 2 active+undersized, 91 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 25/435 objects degraded (5.747%) 2024-06-01T02:45:35.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:35.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:35.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:45:35.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:45:35.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:35.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:35.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:45:35.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:45:35.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:45:35.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:45:35.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-01T02:45:35.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-01T02:45:35.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:34 smithi027 ceph-mon[53801]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-06-01T02:45:36.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:35 smithi003 ceph-mon[67363]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/435 objects degraded (5.747%), 4 pgs degraded) 2024-06-01T02:45:36.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:35 smithi003 ceph-mon[67363]: Cluster is now healthy 2024-06-01T02:45:36.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:35 smithi027 ceph-mon[53801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 25/435 objects degraded (5.747%), 4 pgs degraded) 2024-06-01T02:45:36.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:35 smithi027 ceph-mon[53801]: Cluster is now healthy 2024-06-01T02:45:37.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:36 smithi003 ceph-mon[67363]: pgmap v103: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 153 KiB/s, 0 objects/s recovering 2024-06-01T02:45:37.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:36 smithi027 ceph-mon[53801]: pgmap v103: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 153 KiB/s, 0 objects/s recovering 2024-06-01T02:45:39.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:38 smithi003 ceph-mon[67363]: pgmap v104: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 129 KiB/s, 0 objects/s recovering 2024-06-01T02:45:39.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:38 smithi027 ceph-mon[53801]: pgmap v104: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 129 KiB/s, 0 objects/s recovering 2024-06-01T02:45:41.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:40 smithi003 ceph-mon[67363]: pgmap v105: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 220 B/s rd, 0 op/s; 111 KiB/s, 0 objects/s recovering 2024-06-01T02:45:41.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:40 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:41.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:40 smithi027 ceph-mon[53801]: pgmap v105: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 220 B/s rd, 0 op/s; 111 KiB/s, 0 objects/s recovering 2024-06-01T02:45:41.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:40 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:43.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:42 smithi003 ceph-mon[67363]: pgmap v106: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 199 B/s rd, 0 op/s; 101 KiB/s, 0 objects/s recovering 2024-06-01T02:45:43.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:42 smithi027 ceph-mon[53801]: pgmap v106: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 199 B/s rd, 0 op/s; 101 KiB/s, 0 objects/s recovering 2024-06-01T02:45:45.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:44 smithi003 ceph-mon[67363]: pgmap v107: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:45:45.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:45 smithi027 ceph-mon[53801]: pgmap v107: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:45:47.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:46 smithi003 ceph-mon[67363]: pgmap v108: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:45:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:45:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:46 smithi027 ceph-mon[53801]: pgmap v108: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 86 KiB/s, 0 objects/s recovering 2024-06-01T02:45:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:45:49.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:48 smithi003 ceph-mon[67363]: pgmap v109: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:45:49.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:48 smithi027 ceph-mon[53801]: pgmap v109: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:45:50.010 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:49 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-01T02:45:50.010 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:49 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-01T02:45:50.010 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:49 smithi003 ceph-mon[67363]: Upgrade: osd.3 is safe to restart 2024-06-01T02:45:50.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:49 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-01T02:45:50.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:49 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-01T02:45:50.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:49 smithi027 ceph-mon[53801]: Upgrade: osd.3 is safe to restart 2024-06-01T02:45:51.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:50 smithi003 ceph-mon[67363]: pgmap v110: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:51.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:50 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:51.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:50 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-01T02:45:51.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:50 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:45:51.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:50 smithi027 ceph-mon[53801]: pgmap v110: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:51.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:50 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:45:51.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:50 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-01T02:45:51.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:50 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:45:52.005 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:51 smithi003 ceph-mon[67363]: Upgrade: Updating osd.3 2024-06-01T02:45:52.005 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:51 smithi003 ceph-mon[67363]: Deploying daemon osd.3 on smithi003 2024-06-01T02:45:52.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:51 smithi027 ceph-mon[53801]: Upgrade: Updating osd.3 2024-06-01T02:45:52.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:51 smithi027 ceph-mon[53801]: Deploying daemon osd.3 on smithi003 2024-06-01T02:45:53.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:52 smithi003 ceph-mon[67363]: pgmap v111: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:53.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:52 smithi027 ceph-mon[53801]: pgmap v111: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:54.133 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:53 smithi003 systemd[1]: Stopping Ceph osd.3 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:45:54.590 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:54 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:45:54.132+0000 7fc8aa8dd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:45:54.590 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:54 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:45:54.132+0000 7fc8aa8dd700 -1 osd.3 101 *** Got signal Terminated *** 2024-06-01T02:45:54.590 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:54 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[45922]: 2024-06-01T02:45:54.132+0000 7fc8aa8dd700 -1 osd.3 101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:45:55.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:54 smithi003 ceph-mon[67363]: pgmap v112: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:55.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:54 smithi003 ceph-mon[67363]: osd.3 marked itself down and dead 2024-06-01T02:45:55.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:54 smithi027 ceph-mon[53801]: pgmap v112: 97 pgs: 97 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:45:55.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:54 smithi027 ceph-mon[53801]: osd.3 marked itself down and dead 2024-06-01T02:45:55.781 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:55 smithi003 podman[80846]: 2024-06-01 02:45:55.46075706 +0000 UTC m=+1.453974054 container died e06e99c63f4709c565ff8f9164778da8549b2d02128270d00265e916c0da2762 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, version=8, vendor=Red Hat, Inc., RELEASE=HEAD, name=centos-stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.tags=base centos centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, com.redhat.component=centos-stream-container, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8) 2024-06-01T02:45:55.782 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:55 smithi003 podman[80846]: 2024-06-01 02:45:55.78219155 +0000 UTC m=+1.775408543 container cleanup e06e99c63f4709c565ff8f9164778da8549b2d02128270d00265e916c0da2762 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, RELEASE=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.openshift.expose-services=, io.buildah.version=1.19.8, ceph=True, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-06-01T02:45:56.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:55 smithi003 ceph-mon[67363]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:45:56.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:55 smithi003 ceph-mon[67363]: osdmap e102: 8 total, 7 up, 8 in 2024-06-01T02:45:56.087 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:55 smithi003 bash[80846]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3 2024-06-01T02:45:56.087 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:56 smithi003 podman[80863]: 2024-06-01 02:45:56.0088939 +0000 UTC m=+0.547488890 container remove e06e99c63f4709c565ff8f9164778da8549b2d02128270d00265e916c0da2762 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, distribution-scope=public, vcs-type=git, version=8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-06-01T02:45:56.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:55 smithi027 ceph-mon[53801]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:45:56.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:55 smithi027 ceph-mon[53801]: osdmap e102: 8 total, 7 up, 8 in 2024-06-01T02:45:56.721 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:56 smithi003 podman[80950]: 2024-06-01 02:45:56.459670215 +0000 UTC m=+0.108111981 container create 15814c9b807261f4cb38a2815c013aa7040c6bba71efba694fd92af500a98c95 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True) 2024-06-01T02:45:56.721 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:56 smithi003 podman[80950]: 2024-06-01 02:45:56.370004496 +0000 UTC m=+0.018446283 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:45:56.721 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:56 smithi003 podman[80950]: 2024-06-01 02:45:56.721860064 +0000 UTC m=+0.370301838 container init 15814c9b807261f4cb38a2815c013aa7040c6bba71efba694fd92af500a98c95 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-06-01T02:45:57.041 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:56 smithi003 ceph-mon[67363]: pgmap v114: 97 pgs: 16 stale+active+clean, 81 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:45:57.042 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:56 smithi003 ceph-mon[67363]: osdmap e103: 8 total, 7 up, 8 in 2024-06-01T02:45:57.042 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:56 smithi003 podman[80950]: 2024-06-01 02:45:56.726636993 +0000 UTC m=+0.375078762 container start 15814c9b807261f4cb38a2815c013aa7040c6bba71efba694fd92af500a98c95 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-06-01T02:45:57.042 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:56 smithi003 podman[80950]: 2024-06-01 02:45:56.820797215 +0000 UTC m=+0.469238982 container attach 15814c9b807261f4cb38a2815c013aa7040c6bba71efba694fd92af500a98c95 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True) 2024-06-01T02:45:57.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:56 smithi027 ceph-mon[53801]: pgmap v114: 97 pgs: 16 stale+active+clean, 81 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:45:57.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:56 smithi027 ceph-mon[53801]: osdmap e103: 8 total, 7 up, 8 in 2024-06-01T02:45:57.336 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:57 smithi003 podman[80950]: 2024-06-01 02:45:57.042732904 +0000 UTC m=+0.691174679 container died 15814c9b807261f4cb38a2815c013aa7040c6bba71efba694fd92af500a98c95 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528) 2024-06-01T02:45:57.878 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:57 smithi003 podman[81008]: 2024-06-01 02:45:57.624444535 +0000 UTC m=+0.575409591 container remove 15814c9b807261f4cb38a2815c013aa7040c6bba71efba694fd92af500a98c95 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-06-01T02:45:57.879 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:57 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.3.service: Deactivated successfully. 2024-06-01T02:45:57.879 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:57 smithi003 systemd[1]: Stopped Ceph osd.3 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:45:57.879 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:57 smithi003 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.3.service: Consumed 3min 52.092s CPU time. 2024-06-01T02:45:57.879 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:57 smithi003 systemd[1]: Starting Ceph osd.3 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:45:58.257 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:57 smithi003 ceph-mon[67363]: Health check failed: Degraded data redundancy: 33/435 objects degraded (7.586%), 7 pgs degraded (PG_DEGRADED) 2024-06-01T02:45:58.257 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 podman[81111]: 2024-06-01 02:45:58.0033235 +0000 UTC m=+0.018111710 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:45:58.258 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 podman[81111]: 2024-06-01 02:45:58.12569682 +0000 UTC m=+0.140485016 container create c7d1694d8726bf35d8b6787b70d765a75bf026524597d5d69281029428230128 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0) 2024-06-01T02:45:58.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:57 smithi027 ceph-mon[53801]: Health check failed: Degraded data redundancy: 33/435 objects degraded (7.586%), 7 pgs degraded (PG_DEGRADED) 2024-06-01T02:45:58.586 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 podman[81111]: 2024-06-01 02:45:58.411476927 +0000 UTC m=+0.426265115 container init c7d1694d8726bf35d8b6787b70d765a75bf026524597d5d69281029428230128 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, ceph=True, GIT_CLEAN=True) 2024-06-01T02:45:58.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 podman[81111]: 2024-06-01 02:45:58.415138709 +0000 UTC m=+0.429926914 container start c7d1694d8726bf35d8b6787b70d765a75bf026524597d5d69281029428230128 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:45:58.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 podman[81111]: 2024-06-01 02:45:58.494988901 +0000 UTC m=+0.509777106 container attach c7d1694d8726bf35d8b6787b70d765a75bf026524597d5d69281029428230128 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-06-01T02:45:58.911 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:45:58 smithi003 ceph-mon[67363]: pgmap v116: 97 pgs: 7 active+undersized+degraded, 5 stale+active+clean, 20 active+undersized, 65 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 33/435 objects degraded (7.586%) 2024-06-01T02:45:58.912 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate[81126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-01T02:45:58.912 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate[81126]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-01T02:45:58.912 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 bash[81111]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-01T02:45:58.912 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 bash[81111]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-01T02:45:58.912 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 bash[81111]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-01T02:45:58.912 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 bash[81111]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-01T02:45:58.912 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 bash[81111]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-06-01T02:45:58.912 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 bash[81111]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-01T02:45:58.913 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 bash[81111]: --> ceph-volume raw activate successful for osd ID: 3 2024-06-01T02:45:58.913 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate[81126]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-01T02:45:58.913 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate[81126]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-01T02:45:58.913 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate[81126]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-06-01T02:45:58.913 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate[81126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-01T02:45:58.913 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate[81126]: --> ceph-volume raw activate successful for osd ID: 3 2024-06-01T02:45:58.913 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:58 smithi003 podman[81111]: 2024-06-01 02:45:58.912189477 +0000 UTC m=+0.926977681 container died c7d1694d8726bf35d8b6787b70d765a75bf026524597d5d69281029428230128 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-06-01T02:45:59.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:45:58 smithi027 ceph-mon[53801]: pgmap v116: 97 pgs: 7 active+undersized+degraded, 5 stale+active+clean, 20 active+undersized, 65 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 33/435 objects degraded (7.586%) 2024-06-01T02:46:00.011 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:59 smithi003 podman[81262]: 2024-06-01 02:45:59.656965669 +0000 UTC m=+0.738852757 container remove c7d1694d8726bf35d8b6787b70d765a75bf026524597d5d69281029428230128 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-06-01T02:46:00.011 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:59 smithi003 podman[81322]: 2024-06-01 02:45:59.882901077 +0000 UTC m=+0.113662140 container create 2780cd01c5e61d87f8a9ae602751ab37de65c01b6e7954a7d7d14a1b93ca3ae2 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, GIT_CLEAN=True) 2024-06-01T02:46:00.011 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:45:59 smithi003 podman[81322]: 2024-06-01 02:45:59.788645529 +0000 UTC m=+0.019406623 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:46:00.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:46:00 smithi003 podman[81322]: 2024-06-01 02:46:00.135446316 +0000 UTC m=+0.366207387 container init 2780cd01c5e61d87f8a9ae602751ab37de65c01b6e7954a7d7d14a1b93ca3ae2 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-06-01T02:46:00.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:46:00 smithi003 podman[81322]: 2024-06-01 02:46:00.138903634 +0000 UTC m=+0.369664698 container start 2780cd01c5e61d87f8a9ae602751ab37de65c01b6e7954a7d7d14a1b93ca3ae2 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, ceph=True, GIT_BRANCH=HEAD) 2024-06-01T02:46:00.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:46:00 smithi003 ceph-osd[81341]: -- 172.21.15.3:0/2771274205 <== mon.0 v2:172.21.15.3:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a29cbeb680 con 0x55a29cbb5000 2024-06-01T02:46:00.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:46:00 smithi003 bash[81322]: 2780cd01c5e61d87f8a9ae602751ab37de65c01b6e7954a7d7d14a1b93ca3ae2 2024-06-01T02:46:00.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:46:00 smithi003 systemd[1]: Started Ceph osd.3 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:46:00.677 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:46:01.086 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:46:00 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:46:00.684+0000 7fb302dac7c0 -1 Falling back to public interface 2024-06-01T02:46:01.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:01 smithi003 ceph-mon[67363]: pgmap v117: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:01.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:01.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:01.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:46:01.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:01.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:46:01.433 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:46:01.433 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (4m) 32s ago 28m 14.1M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:46:01.433 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (2m) 32s ago 28m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:46:01.433 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (2m) 2m ago 27m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:46:01.433 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (4m) 32s ago 27m 44.0M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:46:01.433 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (22m) 32s ago 22m 40.7M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:46:01.433 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (22m) 2m ago 22m 18.2M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (4m) 32s ago 29m 507M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (3m) 2m ago 27m 451M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (3m) 32s ago 29m 47.1M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (3m) 2m ago 27m 33.9M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (21m) 32s ago 21m 294M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (21m) 2m ago 21m 88.2M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (4m) 32s ago 27m 8862k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (4m) 2m ago 27m 8418k - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (112s) 32s ago 26m 192M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (75s) 32s ago 26m 174M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (39s) 32s ago 25m 11.5M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:46:01.434 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 starting - - - 1937M 2024-06-01T02:46:01.435 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (24m) 2m ago 24m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:46:01.435 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (24m) 2m ago 24m 461M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:46:01.435 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (24m) 2m ago 24m 375M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:46:01.435 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (23m) 2m ago 23m 618M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:46:01.435 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (3m) 32s ago 26m 67.4M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:46:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:01 smithi027 ceph-mon[53801]: pgmap v117: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:46:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:46:01.998 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:46:01.998 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:46:01.998 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:46:01.998 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:46:01.998 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:46:01.998 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:46:01.998 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 3 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 7 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:46:01.999 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:46:02.284 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:02 smithi003 ceph-mon[67363]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:02.285 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:02 smithi003 ceph-mon[67363]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:02.285 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:02 smithi003 ceph-mon[67363]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:02.285 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:02 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/4033066957' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:02.360 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:46:02.360 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "9/23 daemons upgraded", 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading osd daemons", 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:46:02.361 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:46:02.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:02 smithi027 ceph-mon[53801]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:02.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:02 smithi027 ceph-mon[53801]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:02.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:02 smithi027 ceph-mon[53801]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:02.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:02 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/4033066957' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:02.801 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 61/435 objects degraded (14.023%), 14 pgs degraded 2024-06-01T02:46:02.801 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] OSD_DOWN: 1 osds down 2024-06-01T02:46:02.801 INFO:teuthology.orchestra.run.smithi003.stdout: osd.3 (root=default,host=smithi003) is down 2024-06-01T02:46:02.801 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 61/435 objects degraded (14.023%), 14 pgs degraded 2024-06-01T02:46:02.801 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-06-01T02:46:02.801 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-06-01T02:46:02.801 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-06-01T02:46:02.801 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-06-01T02:46:02.801 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-06-01T02:46:02.802 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.12 is active+undersized+degraded, acting [5,2] 2024-06-01T02:46:02.802 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-06-01T02:46:02.802 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-06-01T02:46:02.802 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-06-01T02:46:02.802 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-06-01T02:46:02.802 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-06-01T02:46:02.802 INFO:teuthology.orchestra.run.smithi003.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-06-01T02:46:02.802 INFO:teuthology.orchestra.run.smithi003.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-06-01T02:46:02.802 INFO:teuthology.orchestra.run.smithi003.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-06-01T02:46:03.540 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:03 smithi003 ceph-mon[67363]: pgmap v118: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:03.541 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:03 smithi003 ceph-mon[67363]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:03.541 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:03 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/452090145' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:46:03.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:03 smithi027 ceph-mon[53801]: pgmap v118: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:03.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:03 smithi027 ceph-mon[53801]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:03.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:03 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/452090145' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:46:05.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:05 smithi003 ceph-mon[67363]: pgmap v119: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:05.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:46:05 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:46:05.471+0000 7fb302dac7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-06-01T02:46:05.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:05 smithi027 ceph-mon[53801]: pgmap v119: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:06.836 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:46:06 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:46:06.574+0000 7fb302dac7c0 -1 osd.3 101 log_to_monitors true 2024-06-01T02:46:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:07 smithi003 ceph-mon[67363]: pgmap v120: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:07 smithi003 ceph-mon[67363]: from='osd.3 [v2:172.21.15.3:6812/2155550336,v1:172.21.15.3:6813/2155550336]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-01T02:46:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:07 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:07 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:07 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 61/435 objects degraded (14.023%), 14 pgs degraded (PG_DEGRADED) 2024-06-01T02:46:07.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:46:07 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:46:07.321+0000 7fb2f4203700 -1 osd.3 101 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:46:07.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:07 smithi027 ceph-mon[53801]: pgmap v120: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:07.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:07 smithi027 ceph-mon[53801]: from='osd.3 [v2:172.21.15.3:6812/2155550336,v1:172.21.15.3:6813/2155550336]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-01T02:46:07.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:07 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:07.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:07 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:07.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:07 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 61/435 objects degraded (14.023%), 14 pgs degraded (PG_DEGRADED) 2024-06-01T02:46:08.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:08 smithi003 ceph-mon[67363]: from='osd.3 [v2:172.21.15.3:6812/2155550336,v1:172.21.15.3:6813/2155550336]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-01T02:46:08.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:08 smithi003 ceph-mon[67363]: osdmap e104: 8 total, 7 up, 8 in 2024-06-01T02:46:08.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:08 smithi003 ceph-mon[67363]: from='osd.3 [v2:172.21.15.3:6812/2155550336,v1:172.21.15.3:6813/2155550336]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:46:08.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:08 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:08.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:08 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:08 smithi027 ceph-mon[53801]: from='osd.3 [v2:172.21.15.3:6812/2155550336,v1:172.21.15.3:6813/2155550336]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-01T02:46:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:08 smithi027 ceph-mon[53801]: osdmap e104: 8 total, 7 up, 8 in 2024-06-01T02:46:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:08 smithi027 ceph-mon[53801]: from='osd.3 [v2:172.21.15.3:6812/2155550336,v1:172.21.15.3:6813/2155550336]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi003", "root=default"]}]: dispatch 2024-06-01T02:46:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:08 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:08.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:08 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:09.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:09 smithi003 ceph-mon[67363]: pgmap v122: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:09.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:09 smithi003 ceph-mon[67363]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:46:09.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:09 smithi003 ceph-mon[67363]: osd.3 [v2:172.21.15.3:6812/2155550336,v1:172.21.15.3:6813/2155550336] boot 2024-06-01T02:46:09.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:09 smithi003 ceph-mon[67363]: osdmap e105: 8 total, 8 up, 8 in 2024-06-01T02:46:09.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:09 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:46:09.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:09 smithi003 ceph-mon[67363]: osdmap e106: 8 total, 8 up, 8 in 2024-06-01T02:46:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:09 smithi027 ceph-mon[53801]: pgmap v122: 97 pgs: 14 active+undersized+degraded, 27 active+undersized, 56 active+clean; 240 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 61/435 objects degraded (14.023%) 2024-06-01T02:46:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:09 smithi027 ceph-mon[53801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:46:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:09 smithi027 ceph-mon[53801]: osd.3 [v2:172.21.15.3:6812/2155550336,v1:172.21.15.3:6813/2155550336] boot 2024-06-01T02:46:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:09 smithi027 ceph-mon[53801]: osdmap e105: 8 total, 8 up, 8 in 2024-06-01T02:46:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:09 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-01T02:46:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:09 smithi027 ceph-mon[53801]: osdmap e106: 8 total, 8 up, 8 in 2024-06-01T02:46:11.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:11 smithi003 ceph-mon[67363]: pgmap v125: 97 pgs: 3 peering, 12 active+undersized+degraded, 26 active+undersized, 56 active+clean; 240 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 49/435 objects degraded (11.264%) 2024-06-01T02:46:11.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:11.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:11.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:46:11.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:46:11.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:11.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:11.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:11 smithi027 ceph-mon[53801]: pgmap v125: 97 pgs: 3 peering, 12 active+undersized+degraded, 26 active+undersized, 56 active+clean; 240 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 49/435 objects degraded (11.264%) 2024-06-01T02:46:11.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:11.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:11.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:46:11.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:46:11.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:11.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:12.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:46:12.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:12.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:12.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:12.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-01T02:46:12.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:12 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-01T02:46:12.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:12 smithi003 ceph-mon[67363]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-01T02:46:12.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:12 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 49/435 objects degraded (11.264%), 12 pgs degraded (PG_DEGRADED) 2024-06-01T02:46:12.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:46:12.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:12.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-01T02:46:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:12 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-01T02:46:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:12 smithi027 ceph-mon[53801]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-01T02:46:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:12 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 49/435 objects degraded (11.264%), 12 pgs degraded (PG_DEGRADED) 2024-06-01T02:46:13.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:13 smithi003 ceph-mon[67363]: pgmap v126: 97 pgs: 3 peering, 10 active+undersized+degraded, 18 active+undersized, 66 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 36/435 objects degraded (8.276%) 2024-06-01T02:46:13.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:13 smithi027 ceph-mon[53801]: pgmap v126: 97 pgs: 3 peering, 10 active+undersized+degraded, 18 active+undersized, 66 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 36/435 objects degraded (8.276%) 2024-06-01T02:46:14.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:14 smithi003 ceph-mon[67363]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/435 objects degraded (8.276%), 10 pgs degraded) 2024-06-01T02:46:14.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:14 smithi003 ceph-mon[67363]: Cluster is now healthy 2024-06-01T02:46:14.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:14 smithi027 ceph-mon[53801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/435 objects degraded (8.276%), 10 pgs degraded) 2024-06-01T02:46:14.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:14 smithi027 ceph-mon[53801]: Cluster is now healthy 2024-06-01T02:46:15.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:15 smithi003 ceph-mon[67363]: pgmap v127: 97 pgs: 3 peering, 94 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 315 B/s rd, 0 op/s 2024-06-01T02:46:15.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:15 smithi027 ceph-mon[53801]: pgmap v127: 97 pgs: 3 peering, 94 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 315 B/s rd, 0 op/s 2024-06-01T02:46:17.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:16 smithi003 ceph-mon[67363]: pgmap v128: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:46:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:46:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:16 smithi027 ceph-mon[53801]: pgmap v128: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:46:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:17.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:46:19.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:18 smithi003 ceph-mon[67363]: pgmap v129: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 215 B/s rd, 0 op/s 2024-06-01T02:46:19.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:18 smithi027 ceph-mon[53801]: pgmap v129: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 215 B/s rd, 0 op/s 2024-06-01T02:46:21.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:20 smithi003 ceph-mon[67363]: pgmap v130: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 195 B/s rd, 0 op/s 2024-06-01T02:46:21.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:20 smithi027 ceph-mon[53801]: pgmap v130: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 195 B/s rd, 0 op/s 2024-06-01T02:46:22.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:22.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:23.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:22 smithi003 ceph-mon[67363]: pgmap v131: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:23.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:22 smithi027 ceph-mon[53801]: pgmap v131: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:25.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:24 smithi003 ceph-mon[67363]: pgmap v132: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:25.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:24 smithi027 ceph-mon[53801]: pgmap v132: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:27.314 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:26 smithi027 ceph-mon[53801]: pgmap v133: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:27.314 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:26 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-01T02:46:27.314 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:26 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-01T02:46:27.314 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:26 smithi027 ceph-mon[53801]: Upgrade: osd.4 is safe to restart 2024-06-01T02:46:27.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:26 smithi003 ceph-mon[67363]: pgmap v133: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:27.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:26 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-01T02:46:27.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:26 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-01T02:46:27.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:26 smithi003 ceph-mon[67363]: Upgrade: osd.4 is safe to restart 2024-06-01T02:46:29.583 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:29 smithi027 ceph-mon[53801]: pgmap v134: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:46:29.584 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:29 smithi027 ceph-mon[53801]: Upgrade: Updating osd.4 2024-06-01T02:46:29.584 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:29.584 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-01T02:46:29.584 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:46:29.584 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:29 smithi027 ceph-mon[53801]: Deploying daemon osd.4 on smithi027 2024-06-01T02:46:29.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:29 smithi003 ceph-mon[67363]: pgmap v134: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:46:29.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:29 smithi003 ceph-mon[67363]: Upgrade: Updating osd.4 2024-06-01T02:46:29.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:29.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-01T02:46:29.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:46:29.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:29 smithi003 ceph-mon[67363]: Deploying daemon osd.4 on smithi027 2024-06-01T02:46:31.315 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:31 smithi027 ceph-mon[53801]: pgmap v135: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:31.315 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:46:31.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:31 smithi003 ceph-mon[67363]: pgmap v135: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:31.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:46:31.943 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:31 smithi027 systemd[1]: Stopping Ceph osd.4 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:46:32.284 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:31 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:46:31.994+0000 7f569ad13700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:46:32.284 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:31 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:46:31.994+0000 7f569ad13700 -1 osd.4 106 *** Got signal Terminated *** 2024-06-01T02:46:32.284 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:31 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[31470]: 2024-06-01T02:46:31.994+0000 7f569ad13700 -1 osd.4 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:46:32.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:32 smithi003 ceph-mon[67363]: osd.4 marked itself down and dead 2024-06-01T02:46:32.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:32 smithi027 ceph-mon[53801]: osd.4 marked itself down and dead 2024-06-01T02:46:33.170 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:46:33.237 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:32 smithi027 podman[58097]: 2024-06-01 02:46:32.911014272 +0000 UTC m=+1.054826122 container died e2ec193ab4e6b71f1f3287de29cce80f614eca6537597ef029930bc0ba6cfb83 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, ceph=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.buildah.version=1.19.8, vendor=Red Hat, Inc., name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, version=8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream) 2024-06-01T02:46:33.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:33 smithi003 ceph-mon[67363]: pgmap v136: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:33.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:33 smithi003 ceph-mon[67363]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:46:33.505 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:33 smithi003 ceph-mon[67363]: osdmap e107: 8 total, 7 up, 8 in 2024-06-01T02:46:33.538 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:33 smithi027 ceph-mon[53801]: pgmap v136: 97 pgs: 97 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:46:33.538 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:33 smithi027 ceph-mon[53801]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:46:33.538 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:33 smithi027 ceph-mon[53801]: osdmap e107: 8 total, 7 up, 8 in 2024-06-01T02:46:33.538 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:33 smithi027 podman[58097]: 2024-06-01 02:46:33.337021159 +0000 UTC m=+1.480832996 container cleanup e2ec193ab4e6b71f1f3287de29cce80f614eca6537597ef029930bc0ba6cfb83 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, RELEASE=HEAD, io.buildah.version=1.19.8, vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754) 2024-06-01T02:46:33.538 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:33 smithi027 bash[58097]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4 2024-06-01T02:46:33.852 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:33 smithi027 podman[58126]: 2024-06-01 02:46:33.538450927 +0000 UTC m=+0.630125987 container remove e2ec193ab4e6b71f1f3287de29cce80f614eca6537597ef029930bc0ba6cfb83 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4, version=8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, release=754, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8) 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (5m) 27s ago 28m 13.7M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (2m) 27s ago 28m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (2m) 2m ago 28m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (5m) 27s ago 28m 44.8M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (22m) 27s ago 22m 40.7M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (22m) 2m ago 22m 18.2M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (4m) 27s ago 29m 508M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (4m) 2m ago 27m 451M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (3m) 27s ago 29m 50.2M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:46:33.885 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (3m) 2m ago 27m 33.9M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (22m) 27s ago 22m 295M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (22m) 2m ago 22m 88.2M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (4m) 27s ago 28m 8702k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (4m) 2m ago 27m 8418k - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (2m) 27s ago 27m 194M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (108s) 27s ago 26m 175M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (71s) 27s ago 26m 147M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (34s) 27s ago 25m 15.0M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (25m) 2m ago 25m 606M 2704M 17.2.0 e1d6a67b021e e2ec193ab4e6 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (25m) 2m ago 25m 461M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (24m) 2m ago 24m 375M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:46:33.886 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (24m) 2m ago 24m 618M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:46:33.887 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (4m) 27s ago 27m 67.7M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:46:34.241 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:33 smithi027 podman[58215]: 2024-06-01 02:46:33.964619225 +0000 UTC m=+0.113350290 container create aa853d595d7f7fd803fb1cf295a756d01f3fa1cf6e403e41c7f63a9d27ec3fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-06-01T02:46:34.242 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:33 smithi027 podman[58215]: 2024-06-01 02:46:33.869033434 +0000 UTC m=+0.017764507 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:46:34.242 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:34 smithi027 podman[58215]: 2024-06-01 02:46:34.242077846 +0000 UTC m=+0.390808912 container init aa853d595d7f7fd803fb1cf295a756d01f3fa1cf6e403e41c7f63a9d27ec3fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:46:34.297 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:46:34.297 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 4 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:46:34.298 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:46:34.299 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:46:34.299 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:46:34.299 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-06-01T02:46:34.299 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 8 2024-06-01T02:46:34.299 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:46:34.299 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:46:34.551 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:34 smithi027 podman[58215]: 2024-06-01 02:46:34.245769142 +0000 UTC m=+0.394500207 container start aa853d595d7f7fd803fb1cf295a756d01f3fa1cf6e403e41c7f63a9d27ec3fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2) 2024-06-01T02:46:34.551 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:34 smithi027 podman[58215]: 2024-06-01 02:46:34.308573901 +0000 UTC m=+0.457304986 container attach aa853d595d7f7fd803fb1cf295a756d01f3fa1cf6e403e41c7f63a9d27ec3fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528) 2024-06-01T02:46:34.551 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:34 smithi027 ceph-mon[53801]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:34.551 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:34 smithi027 ceph-mon[53801]: osdmap e108: 8 total, 7 up, 8 in 2024-06-01T02:46:34.552 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:34 smithi027 ceph-mon[53801]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:34 smithi003 ceph-mon[67363]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:34 smithi003 ceph-mon[67363]: osdmap e108: 8 total, 7 up, 8 in 2024-06-01T02:46:34.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:34 smithi003 ceph-mon[67363]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:34.648 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:46:34.648 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:46:34.648 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:46:34.648 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:46:34.648 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:46:34.648 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:46:34.648 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:46:34.649 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:46:34.649 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:46:34.649 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "10/23 daemons upgraded", 2024-06-01T02:46:34.649 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading osd daemons", 2024-06-01T02:46:34.649 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:46:34.649 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:46:34.847 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:34 smithi027 podman[58215]: 2024-06-01 02:46:34.551775228 +0000 UTC m=+0.700506296 container died aa853d595d7f7fd803fb1cf295a756d01f3fa1cf6e403e41c7f63a9d27ec3fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True) 2024-06-01T02:46:35.063 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_WARN 1 osds down; Reduced data availability: 3 pgs peering 2024-06-01T02:46:35.063 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] OSD_DOWN: 1 osds down 2024-06-01T02:46:35.063 INFO:teuthology.orchestra.run.smithi003.stdout: osd.4 (root=default,host=smithi027) is down 2024-06-01T02:46:35.063 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] PG_AVAILABILITY: Reduced data availability: 3 pgs peering 2024-06-01T02:46:35.064 INFO:teuthology.orchestra.run.smithi003.stdout: pg 2.f is stuck peering for 15m, current state peering, last acting [0,7] 2024-06-01T02:46:35.064 INFO:teuthology.orchestra.run.smithi003.stdout: pg 3.3 is stuck peering for 10m, current state peering, last acting [0,6] 2024-06-01T02:46:35.064 INFO:teuthology.orchestra.run.smithi003.stdout: pg 3.1a is stuck peering for 10m, current state peering, last acting [1,2] 2024-06-01T02:46:35.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:35 smithi003 ceph-mon[67363]: pgmap v139: 97 pgs: 20 peering, 7 stale+active+clean, 70 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:46:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:35 smithi003 ceph-mon[67363]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:35 smithi003 ceph-mon[67363]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-06-01T02:46:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:35 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3894893844' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:35.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:35 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3323986922' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:46:35.580 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:35 smithi027 ceph-mon[53801]: pgmap v139: 97 pgs: 20 peering, 7 stale+active+clean, 70 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:46:35.580 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:35 smithi027 ceph-mon[53801]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:35.580 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:35 smithi027 ceph-mon[53801]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-06-01T02:46:35.580 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:35 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3894893844' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:35.580 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:35 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3323986922' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:46:35.580 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 podman[58241]: 2024-06-01 02:46:35.122744689 +0000 UTC m=+0.564161843 container remove aa853d595d7f7fd803fb1cf295a756d01f3fa1cf6e403e41c7f63a9d27ec3fb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-06-01T02:46:35.580 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.4.service: Deactivated successfully. 2024-06-01T02:46:35.580 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 systemd[1]: Stopped Ceph osd.4 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:46:35.580 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.4.service: Consumed 3min 23.189s CPU time. 2024-06-01T02:46:35.581 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 systemd[1]: Starting Ceph osd.4 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:46:35.842 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 podman[58344]: 2024-06-01 02:46:35.483071201 +0000 UTC m=+0.018107586 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:46:35.842 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 podman[58344]: 2024-06-01 02:46:35.61565424 +0000 UTC m=+0.150690667 container create 044a08cf4a6f56276f49d2b909a4915188d850423322ae4dc29e519b083f0898 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5) 2024-06-01T02:46:35.843 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 podman[58344]: 2024-06-01 02:46:35.84300194 +0000 UTC m=+0.378038308 container init 044a08cf4a6f56276f49d2b909a4915188d850423322ae4dc29e519b083f0898 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate, ceph=True, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-06-01T02:46:36.102 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 podman[58344]: 2024-06-01 02:46:35.846846005 +0000 UTC m=+0.381882370 container start 044a08cf4a6f56276f49d2b909a4915188d850423322ae4dc29e519b083f0898 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0) 2024-06-01T02:46:36.102 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:35 smithi027 podman[58344]: 2024-06-01 02:46:35.95143413 +0000 UTC m=+0.486470496 container attach 044a08cf4a6f56276f49d2b909a4915188d850423322ae4dc29e519b083f0898 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-06-01T02:46:36.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:36 smithi003 ceph-mon[67363]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:36.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:36 smithi027 ceph-mon[53801]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:46:36.602 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate[58359]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate[58359]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 bash[58344]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 bash[58344]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 bash[58344]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 bash[58344]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 bash[58344]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 bash[58344]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 bash[58344]: --> ceph-volume raw activate successful for osd ID: 4 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate[58359]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate[58359]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-01T02:46:36.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate[58359]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-06-01T02:46:36.604 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate[58359]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-01T02:46:36.604 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate[58359]: --> ceph-volume raw activate successful for osd ID: 4 2024-06-01T02:46:36.604 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 podman[58344]: 2024-06-01 02:46:36.335764273 +0000 UTC m=+0.870800639 container died 044a08cf4a6f56276f49d2b909a4915188d850423322ae4dc29e519b083f0898 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, io.buildah.version=1.33.5) 2024-06-01T02:46:37.172 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:36 smithi027 podman[58485]: 2024-06-01 02:46:36.922073523 +0000 UTC m=+0.576728693 container remove 044a08cf4a6f56276f49d2b909a4915188d850423322ae4dc29e519b083f0898 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-activate, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-06-01T02:46:37.172 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:37 smithi027 podman[58575]: 2024-06-01 02:46:37.046593603 +0000 UTC m=+0.017926371 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:46:37.172 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:37 smithi027 podman[58575]: 2024-06-01 02:46:37.1731871 +0000 UTC m=+0.144519838 container create a133545ce0d3b1a05882a17a41d32e9dbbddc2b121b3a12e8ca86128b035b306 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-06-01T02:46:37.425 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:37 smithi027 ceph-mon[53801]: pgmap v140: 97 pgs: 20 peering, 5 stale+active+clean, 2 active+undersized, 70 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-06-01T02:46:37.425 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:37 smithi027 podman[58575]: 2024-06-01 02:46:37.400804717 +0000 UTC m=+0.372137475 container init a133545ce0d3b1a05882a17a41d32e9dbbddc2b121b3a12e8ca86128b035b306 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528) 2024-06-01T02:46:37.425 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:37 smithi027 podman[58575]: 2024-06-01 02:46:37.40417851 +0000 UTC m=+0.375511254 container start a133545ce0d3b1a05882a17a41d32e9dbbddc2b121b3a12e8ca86128b035b306 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0) 2024-06-01T02:46:37.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:37 smithi003 ceph-mon[67363]: pgmap v140: 97 pgs: 20 peering, 5 stale+active+clean, 2 active+undersized, 70 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-06-01T02:46:37.852 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:37 smithi027 ceph-osd[58594]: -- 172.21.15.27:0/682877886 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5607bb9e3680 con 0x5607bb9ad000 2024-06-01T02:46:37.852 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:37 smithi027 bash[58575]: a133545ce0d3b1a05882a17a41d32e9dbbddc2b121b3a12e8ca86128b035b306 2024-06-01T02:46:37.852 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:37 smithi027 systemd[1]: Started Ceph osd.4 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:46:38.339 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:37 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:46:37.946+0000 7f88c618f7c0 -1 Falling back to public interface 2024-06-01T02:46:38.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:38 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:38.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:38 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:38.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:38 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:46:38.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:38 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:38 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:38 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:46:39.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:39 smithi003 ceph-mon[67363]: pgmap v141: 97 pgs: 8 active+undersized+degraded, 7 peering, 26 active+undersized, 56 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 31/435 objects degraded (7.126%) 2024-06-01T02:46:39.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:39 smithi003 ceph-mon[67363]: Health check failed: Degraded data redundancy: 31/435 objects degraded (7.126%), 8 pgs degraded (PG_DEGRADED) 2024-06-01T02:46:39.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:39 smithi027 ceph-mon[53801]: pgmap v141: 97 pgs: 8 active+undersized+degraded, 7 peering, 26 active+undersized, 56 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 31/435 objects degraded (7.126%) 2024-06-01T02:46:39.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:39 smithi027 ceph-mon[53801]: Health check failed: Degraded data redundancy: 31/435 objects degraded (7.126%), 8 pgs degraded (PG_DEGRADED) 2024-06-01T02:46:40.867 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:40 smithi027 ceph-mon[53801]: pgmap v142: 97 pgs: 10 active+undersized+degraded, 31 active+undersized, 56 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:46:40.867 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:40 smithi027 ceph-mon[53801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-01T02:46:41.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:40 smithi003 ceph-mon[67363]: pgmap v142: 97 pgs: 10 active+undersized+degraded, 31 active+undersized, 56 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:46:41.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:40 smithi003 ceph-mon[67363]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-01T02:46:42.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:42 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:42.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:42 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:42.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:42 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:42.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:42 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:43 smithi027 ceph-mon[53801]: pgmap v143: 97 pgs: 10 active+undersized+degraded, 31 active+undersized, 56 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 215 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:46:43.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:43 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:43.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:43 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:43.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:43 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:46:43.376+0000 7f88c618f7c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-06-01T02:46:43.689 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:43 smithi003 ceph-mon[67363]: pgmap v143: 97 pgs: 10 active+undersized+degraded, 31 active+undersized, 56 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 215 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:46:43.689 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:43 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:43.689 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:43 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:44.650 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:44 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:46:44.407+0000 7f88c618f7c0 -1 osd.4 106 log_to_monitors true 2024-06-01T02:46:45.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:45 smithi027 ceph-mon[53801]: pgmap v144: 97 pgs: 10 active+undersized+degraded, 31 active+undersized, 56 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 194 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:46:45.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:45 smithi027 ceph-mon[53801]: from='osd.4 [v2:172.21.15.27:6800/1990922986,v1:172.21.15.27:6801/1990922986]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-01T02:46:45.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:45 smithi027 ceph-mon[53801]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-01T02:46:45.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:45 smithi003 ceph-mon[67363]: pgmap v144: 97 pgs: 10 active+undersized+degraded, 31 active+undersized, 56 active+clean; 240 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 194 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:46:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:45 smithi003 ceph-mon[67363]: from='osd.4 [v2:172.21.15.27:6800/1990922986,v1:172.21.15.27:6801/1990922986]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-01T02:46:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:45 smithi003 ceph-mon[67363]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-01T02:46:46.102 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:46:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:46:45.849+0000 7f88b35de700 -1 osd.4 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:46:46.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-01T02:46:46.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: osdmap e109: 8 total, 7 up, 8 in 2024-06-01T02:46:46.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='osd.4 [v2:172.21.15.27:6800/1990922986,v1:172.21.15.27:6801/1990922986]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:46:46.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:46:46.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:46:46.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='osd.4 ' entity='osd.4' 2024-06-01T02:46:46.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:46.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: osdmap e109: 8 total, 7 up, 8 in 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='osd.4 [v2:172.21.15.27:6800/1990922986,v1:172.21.15.27:6801/1990922986]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='osd.4 ' entity='osd.4' 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:46:46.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:46:46.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:46.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:46.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:46:46.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-01T02:46:47.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:47 smithi027 ceph-mon[53801]: pgmap v146: 97 pgs: 10 active+undersized+degraded, 31 active+undersized, 56 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:46:47.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:47 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-01T02:46:47.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:47 smithi027 ceph-mon[53801]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-01T02:46:47.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:47 smithi027 ceph-mon[53801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:46:47.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:47 smithi027 ceph-mon[53801]: osd.4 [v2:172.21.15.27:6800/1990922986,v1:172.21.15.27:6801/1990922986] boot 2024-06-01T02:46:47.754 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:47 smithi027 ceph-mon[53801]: osdmap e110: 8 total, 8 up, 8 in 2024-06-01T02:46:47.755 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:47 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:46:47.755 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:47 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 40/435 objects degraded (9.195%), 10 pgs degraded (PG_DEGRADED) 2024-06-01T02:46:47.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:47 smithi003 ceph-mon[67363]: pgmap v146: 97 pgs: 10 active+undersized+degraded, 31 active+undersized, 56 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:46:47.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:47 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-01T02:46:47.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:47 smithi003 ceph-mon[67363]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-06-01T02:46:47.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:47 smithi003 ceph-mon[67363]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:46:47.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:47 smithi003 ceph-mon[67363]: osd.4 [v2:172.21.15.27:6800/1990922986,v1:172.21.15.27:6801/1990922986] boot 2024-06-01T02:46:47.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:47 smithi003 ceph-mon[67363]: osdmap e110: 8 total, 8 up, 8 in 2024-06-01T02:46:47.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:47 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-01T02:46:47.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:47 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 40/435 objects degraded (9.195%), 10 pgs degraded (PG_DEGRADED) 2024-06-01T02:46:49.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:48 smithi027 ceph-mon[53801]: pgmap v148: 97 pgs: 8 peering, 8 active+undersized+degraded, 25 active+undersized, 56 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 27/435 objects degraded (6.207%) 2024-06-01T02:46:49.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:48 smithi027 ceph-mon[53801]: osdmap e111: 8 total, 8 up, 8 in 2024-06-01T02:46:49.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:48 smithi003 ceph-mon[67363]: pgmap v148: 97 pgs: 8 peering, 8 active+undersized+degraded, 25 active+undersized, 56 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 27/435 objects degraded (6.207%) 2024-06-01T02:46:49.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:48 smithi003 ceph-mon[67363]: osdmap e111: 8 total, 8 up, 8 in 2024-06-01T02:46:51.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:50 smithi003 ceph-mon[67363]: pgmap v150: 97 pgs: 8 peering, 6 active+undersized+degraded, 22 active+undersized, 61 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 18/435 objects degraded (4.138%) 2024-06-01T02:46:51.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:50 smithi027 ceph-mon[53801]: pgmap v150: 97 pgs: 8 peering, 6 active+undersized+degraded, 22 active+undersized, 61 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 18/435 objects degraded (4.138%) 2024-06-01T02:46:52.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:51 smithi003 ceph-mon[67363]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/435 objects degraded (4.138%), 6 pgs degraded) 2024-06-01T02:46:52.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:51 smithi003 ceph-mon[67363]: Cluster is now healthy 2024-06-01T02:46:52.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:51 smithi027 ceph-mon[53801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/435 objects degraded (4.138%), 6 pgs degraded) 2024-06-01T02:46:52.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:51 smithi027 ceph-mon[53801]: Cluster is now healthy 2024-06-01T02:46:53.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:52 smithi003 ceph-mon[67363]: pgmap v151: 97 pgs: 8 peering, 89 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 320 B/s rd, 0 op/s 2024-06-01T02:46:53.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:52 smithi027 ceph-mon[53801]: pgmap v151: 97 pgs: 8 peering, 89 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 320 B/s rd, 0 op/s 2024-06-01T02:46:55.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:54 smithi003 ceph-mon[67363]: pgmap v152: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:46:55.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:54 smithi027 ceph-mon[53801]: pgmap v152: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:46:57.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:56 smithi003 ceph-mon[67363]: pgmap v153: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 0 op/s 2024-06-01T02:46:57.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:56 smithi027 ceph-mon[53801]: pgmap v153: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 228 B/s rd, 0 op/s 2024-06-01T02:46:59.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:46:58 smithi003 ceph-mon[67363]: pgmap v154: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:46:59.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:46:58 smithi027 ceph-mon[53801]: pgmap v154: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:47:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:00 smithi003 ceph-mon[67363]: pgmap v155: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 0 op/s 2024-06-01T02:47:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:47:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:00 smithi027 ceph-mon[53801]: pgmap v155: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 0 op/s 2024-06-01T02:47:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:47:01.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:02.223 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-01T02:47:02.223 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:01 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-01T02:47:02.224 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:01 smithi027 ceph-mon[53801]: Upgrade: osd.5 is safe to restart 2024-06-01T02:47:02.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-01T02:47:02.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:01 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-01T02:47:02.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:01 smithi003 ceph-mon[67363]: Upgrade: osd.5 is safe to restart 2024-06-01T02:47:03.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:02 smithi003 ceph-mon[67363]: pgmap v156: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:03.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:02 smithi027 ceph-mon[53801]: pgmap v156: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:04.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:03 smithi027 ceph-mon[53801]: Upgrade: Updating osd.5 2024-06-01T02:47:04.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:03 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:04.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:03 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-01T02:47:04.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:03 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:47:04.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:03 smithi027 ceph-mon[53801]: Deploying daemon osd.5 on smithi027 2024-06-01T02:47:04.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:03 smithi003 ceph-mon[67363]: Upgrade: Updating osd.5 2024-06-01T02:47:04.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:03 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:04.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:03 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-01T02:47:04.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:03 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:47:04.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:03 smithi003 ceph-mon[67363]: Deploying daemon osd.5 on smithi027 2024-06-01T02:47:05.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:04 smithi027 ceph-mon[53801]: pgmap v157: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:05.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:04 smithi003 ceph-mon[67363]: pgmap v157: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:05.440 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:47:06.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:05 smithi027 ceph-mon[53801]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:06.127 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (5m) 59s ago 29m 13.7M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (3m) 59s ago 29m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (3m) 24s ago 28m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (5m) 59s ago 28m 44.8M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (23m) 59s ago 23m 40.7M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (23m) 24s ago 23m 18.8M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (5m) 59s ago 30m 508M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (4m) 24s ago 28m 453M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (4m) 59s ago 30m 50.2M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (4m) 24s ago 28m 38.7M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (23m) 59s ago 22m 295M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (22m) 24s ago 22m 88.5M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:47:06.128 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (5m) 59s ago 28m 8702k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (5m) 24s ago 28m 11.0M - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (2m) 59s ago 27m 194M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (2m) 59s ago 27m 175M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (103s) 59s ago 26m 147M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (66s) 59s ago 26m 15.0M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (29s) 24s ago 25m 14.8M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (25m) 24s ago 25m 461M 2704M 17.2.0 e1d6a67b021e 01a3b7de6f1f 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (25m) 24s ago 25m 374M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (24m) 24s ago 24m 616M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:47:06.129 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (4m) 59s ago 28m 67.7M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:47:06.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:05 smithi003 ceph-mon[67363]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:06.352 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:06 smithi027 systemd[1]: Stopping Ceph osd.5 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:47:06.540 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:47:06.540 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:47:06.540 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:47:06.540 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 5 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-06-01T02:47:06.541 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 9 2024-06-01T02:47:06.542 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:47:06.542 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:47:06.852 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:47:06.379+0000 7fe2a2a3e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:47:06.852 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:47:06.379+0000 7fe2a2a3e700 -1 osd.5 111 *** Got signal Terminated *** 2024-06-01T02:47:06.853 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[34433]: 2024-06-01T02:47:06.379+0000 7fe2a2a3e700 -1 osd.5 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:47:06.890 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:47:06.890 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:47:06.890 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:47:06.890 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:47:06.890 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:47:06.890 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:47:06.890 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:47:06.890 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:47:06.891 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:47:06.891 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "11/23 daemons upgraded", 2024-06-01T02:47:06.891 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading osd daemons", 2024-06-01T02:47:06.891 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:47:06.891 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:47:07.310 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_WARN 1 osds down 2024-06-01T02:47:07.310 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] OSD_DOWN: 1 osds down 2024-06-01T02:47:07.310 INFO:teuthology.orchestra.run.smithi003.stdout: osd.5 (root=default,host=smithi027) is down 2024-06-01T02:47:07.310 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:06 smithi003 ceph-mon[67363]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:07.310 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:06 smithi003 ceph-mon[67363]: pgmap v158: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:07.310 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:06 smithi003 ceph-mon[67363]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:07.311 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:06 smithi003 ceph-mon[67363]: osd.5 marked itself down and dead 2024-06-01T02:47:07.311 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:06 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1424142293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:07.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:06 smithi027 ceph-mon[53801]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:07.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:06 smithi027 ceph-mon[53801]: pgmap v158: 97 pgs: 97 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:07.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:06 smithi027 ceph-mon[53801]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:07.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:06 smithi027 ceph-mon[53801]: osd.5 marked itself down and dead 2024-06-01T02:47:07.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:06 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/1424142293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:07.713 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:07 smithi027 podman[60403]: 2024-06-01 02:47:07.442661235 +0000 UTC m=+1.177470712 container died 01a3b7de6f1fb358b5f7fd9e296984d72f3ffae2056c2e556375fc457f7a3c44 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, name=centos-stream, architecture=x86_64, ceph=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , release=754, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0) 2024-06-01T02:47:08.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:07 smithi027 ceph-mon[53801]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:08.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:07 smithi027 ceph-mon[53801]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:47:08.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:07 smithi027 ceph-mon[53801]: osdmap e112: 8 total, 7 up, 8 in 2024-06-01T02:47:08.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:07 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/425424288' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:47:08.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:07 smithi027 podman[60403]: 2024-06-01 02:47:07.787660632 +0000 UTC m=+1.522470108 container cleanup 01a3b7de6f1fb358b5f7fd9e296984d72f3ffae2056c2e556375fc457f7a3c44 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, version=8, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870) 2024-06-01T02:47:08.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:07 smithi027 bash[60403]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5 2024-06-01T02:47:08.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:07 smithi003 ceph-mon[67363]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:08.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:07 smithi003 ceph-mon[67363]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:47:08.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:07 smithi003 ceph-mon[67363]: osdmap e112: 8 total, 7 up, 8 in 2024-06-01T02:47:08.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:07 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/425424288' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:47:08.447 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:08 smithi027 podman[60421]: 2024-06-01 02:47:08.105589637 +0000 UTC m=+0.662618719 container remove 01a3b7de6f1fb358b5f7fd9e296984d72f3ffae2056c2e556375fc457f7a3c44 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-type=git, maintainer=Guillaume Abrioux , release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.buildah.version=1.19.8, GIT_CLEAN=True, RELEASE=HEAD, architecture=x86_64) 2024-06-01T02:47:08.783 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:08 smithi027 podman[60511]: 2024-06-01 02:47:08.448254565 +0000 UTC m=+0.091803347 container create bb7a840e71e73cce6415af232ac486c93e6e6d8d0082dfd268e2758797cd47bd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-3d89acc, ceph=True, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-06-01T02:47:08.783 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:08 smithi027 podman[60511]: 2024-06-01 02:47:08.374092567 +0000 UTC m=+0.017641378 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:47:08.783 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:08 smithi027 podman[60511]: 2024-06-01 02:47:08.683856715 +0000 UTC m=+0.327405496 container init bb7a840e71e73cce6415af232ac486c93e6e6d8d0082dfd268e2758797cd47bd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux ) 2024-06-01T02:47:08.783 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:08 smithi027 podman[60511]: 2024-06-01 02:47:08.687228614 +0000 UTC m=+0.330777404 container start bb7a840e71e73cce6415af232ac486c93e6e6d8d0082dfd268e2758797cd47bd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-06-01T02:47:08.784 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:08 smithi027 podman[60511]: 2024-06-01 02:47:08.783923787 +0000 UTC m=+0.427472590 container attach bb7a840e71e73cce6415af232ac486c93e6e6d8d0082dfd268e2758797cd47bd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-06-01T02:47:09.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:08 smithi027 ceph-mon[53801]: pgmap v160: 97 pgs: 8 peering, 9 stale+active+clean, 80 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:47:09.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:08 smithi027 ceph-mon[53801]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-06-01T02:47:09.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:08 smithi027 ceph-mon[53801]: osdmap e113: 8 total, 7 up, 8 in 2024-06-01T02:47:09.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:08 smithi027 podman[60511]: 2024-06-01 02:47:08.990275255 +0000 UTC m=+0.633824045 container died bb7a840e71e73cce6415af232ac486c93e6e6d8d0082dfd268e2758797cd47bd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:47:09.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:08 smithi003 ceph-mon[67363]: pgmap v160: 97 pgs: 8 peering, 9 stale+active+clean, 80 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:47:09.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:08 smithi003 ceph-mon[67363]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-06-01T02:47:09.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:08 smithi003 ceph-mon[67363]: osdmap e113: 8 total, 7 up, 8 in 2024-06-01T02:47:09.590 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:09 smithi027 podman[60537]: 2024-06-01 02:47:09.587727336 +0000 UTC m=+0.590870751 container remove bb7a840e71e73cce6415af232ac486c93e6e6d8d0082dfd268e2758797cd47bd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528) 2024-06-01T02:47:10.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:09 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.5.service: Deactivated successfully. 2024-06-01T02:47:10.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:09 smithi027 systemd[1]: Stopped Ceph osd.5 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:47:10.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:09 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.5.service: Consumed 3min 45.603s CPU time. 2024-06-01T02:47:10.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:09 smithi027 systemd[1]: Starting Ceph osd.5 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:47:10.274 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:10 smithi027 ceph-mon[53801]: Health check failed: Degraded data redundancy: 16/435 objects degraded (3.678%), 4 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:10.274 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 podman[60638]: 2024-06-01 02:47:09.947593252 +0000 UTC m=+0.017714269 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:47:10.274 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 podman[60638]: 2024-06-01 02:47:10.06380356 +0000 UTC m=+0.133924565 container create 5cd9a24d7760bff84bb018fb70abc91f0a10ead2d667aa776d5629b606706172 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:47:10.274 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 podman[60638]: 2024-06-01 02:47:10.274494977 +0000 UTC m=+0.344615985 container init 5cd9a24d7760bff84bb018fb70abc91f0a10ead2d667aa776d5629b606706172 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-06-01T02:47:10.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:10 smithi003 ceph-mon[67363]: Health check failed: Degraded data redundancy: 16/435 objects degraded (3.678%), 4 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:10.602 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 podman[60638]: 2024-06-01 02:47:10.278382498 +0000 UTC m=+0.348503519 container start 5cd9a24d7760bff84bb018fb70abc91f0a10ead2d667aa776d5629b606706172 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240528) 2024-06-01T02:47:10.603 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 podman[60638]: 2024-06-01 02:47:10.357758009 +0000 UTC m=+0.427879027 container attach 5cd9a24d7760bff84bb018fb70abc91f0a10ead2d667aa776d5629b606706172 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate, RELEASE=squid-3d89acc, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux ) 2024-06-01T02:47:11.002 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate[60654]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-01T02:47:11.002 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate[60654]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-01T02:47:11.002 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 bash[60638]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-01T02:47:11.002 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 bash[60638]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 bash[60638]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 bash[60638]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 bash[60638]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 bash[60638]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 bash[60638]: --> ceph-volume raw activate successful for osd ID: 5 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate[60654]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate[60654]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate[60654]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate[60654]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate[60654]: --> ceph-volume raw activate successful for osd ID: 5 2024-06-01T02:47:11.003 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:10 smithi027 podman[60638]: 2024-06-01 02:47:10.775372528 +0000 UTC m=+0.845493543 container died 5cd9a24d7760bff84bb018fb70abc91f0a10ead2d667aa776d5629b606706172 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc) 2024-06-01T02:47:11.336 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:11 smithi027 ceph-mon[53801]: pgmap v162: 97 pgs: 4 active+undersized+degraded, 8 peering, 7 stale+active+clean, 4 active+undersized, 74 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 16/435 objects degraded (3.678%) 2024-06-01T02:47:11.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:11 smithi003 ceph-mon[67363]: pgmap v162: 97 pgs: 4 active+undersized+degraded, 8 peering, 7 stale+active+clean, 4 active+undersized, 74 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 16/435 objects degraded (3.678%) 2024-06-01T02:47:11.602 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:11 smithi027 podman[60769]: 2024-06-01 02:47:11.336566478 +0000 UTC m=+0.554903315 container remove 5cd9a24d7760bff84bb018fb70abc91f0a10ead2d667aa776d5629b606706172 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0) 2024-06-01T02:47:11.603 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:11 smithi027 podman[60830]: 2024-06-01 02:47:11.459331587 +0000 UTC m=+0.017930514 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:47:11.603 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:11 smithi027 podman[60830]: 2024-06-01 02:47:11.579334604 +0000 UTC m=+0.137933573 container create cce965464b24de85407a228e021894f5b73c0a6cfdc7af6b96e63ebf8652455f (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=) 2024-06-01T02:47:12.102 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:11 smithi027 podman[60830]: 2024-06-01 02:47:11.831918268 +0000 UTC m=+0.390517195 container init cce965464b24de85407a228e021894f5b73c0a6cfdc7af6b96e63ebf8652455f (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, ceph=True) 2024-06-01T02:47:12.102 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:11 smithi027 podman[60830]: 2024-06-01 02:47:11.835649677 +0000 UTC m=+0.394248604 container start cce965464b24de85407a228e021894f5b73c0a6cfdc7af6b96e63ebf8652455f (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:47:12.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:11 smithi027 ceph-osd[60850]: -- 172.21.15.27:0/3060754379 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56130f52b680 con 0x56130f4f5000 2024-06-01T02:47:12.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:11 smithi027 bash[60830]: cce965464b24de85407a228e021894f5b73c0a6cfdc7af6b96e63ebf8652455f 2024-06-01T02:47:12.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:11 smithi027 systemd[1]: Started Ceph osd.5 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:47:12.852 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:12 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:47:12.376+0000 7effac8087c0 -1 Falling back to public interface 2024-06-01T02:47:13.336 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:12 smithi003 ceph-mon[67363]: pgmap v163: 97 pgs: 10 active+undersized+degraded, 8 peering, 13 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:47:13.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:13.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:13.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:47:13.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:12 smithi027 ceph-mon[53801]: pgmap v163: 97 pgs: 10 active+undersized+degraded, 8 peering, 13 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 40/435 objects degraded (9.195%) 2024-06-01T02:47:13.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:13.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:13.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:47:14.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:14 smithi003 ceph-mon[67363]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-06-01T02:47:14.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:14 smithi027 ceph-mon[53801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-06-01T02:47:15.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:15 smithi003 ceph-mon[67363]: pgmap v164: 97 pgs: 12 active+undersized+degraded, 19 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 46/435 objects degraded (10.575%) 2024-06-01T02:47:15.344 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:15 smithi027 ceph-mon[53801]: pgmap v164: 97 pgs: 12 active+undersized+degraded, 19 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 46/435 objects degraded (10.575%) 2024-06-01T02:47:16.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:16.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:16.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:16 smithi027 ceph-mon[53801]: pgmap v165: 97 pgs: 12 active+undersized+degraded, 19 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 231 B/s rd, 0 op/s; 46/435 objects degraded (10.575%) 2024-06-01T02:47:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:47:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:16 smithi003 ceph-mon[67363]: pgmap v165: 97 pgs: 12 active+undersized+degraded, 19 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 231 B/s rd, 0 op/s; 46/435 objects degraded (10.575%) 2024-06-01T02:47:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:17.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:47:18.042 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:17 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:18.043 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:17 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:18.043 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:17 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 46/435 objects degraded (10.575%), 12 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:18.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:17 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:18.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:17 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:18.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:17 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 46/435 objects degraded (10.575%), 12 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:18.352 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:18 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:47:18.041+0000 7effac8087c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-06-01T02:47:18.975 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:18 smithi027 ceph-mon[53801]: pgmap v166: 97 pgs: 12 active+undersized+degraded, 19 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 46/435 objects degraded (10.575%) 2024-06-01T02:47:19.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:18 smithi003 ceph-mon[67363]: pgmap v166: 97 pgs: 12 active+undersized+degraded, 19 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 46/435 objects degraded (10.575%) 2024-06-01T02:47:19.352 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:19 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:47:19.168+0000 7effac8087c0 -1 osd.5 111 log_to_monitors true 2024-06-01T02:47:19.986 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:19 smithi027 ceph-mon[53801]: from='osd.5 [v2:172.21.15.27:6808/2990360884,v1:172.21.15.27:6809/2990360884]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-01T02:47:19.986 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:19 smithi027 ceph-mon[53801]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-01T02:47:20.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:19 smithi003 ceph-mon[67363]: from='osd.5 [v2:172.21.15.27:6808/2990360884,v1:172.21.15.27:6809/2990360884]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-01T02:47:20.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:19 smithi003 ceph-mon[67363]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-01T02:47:20.602 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:47:20.247+0000 7eff99c57700 -1 osd.5 111 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:47:21.091 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-01T02:47:21.091 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: osdmap e114: 8 total, 7 up, 8 in 2024-06-01T02:47:21.091 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='osd.5 [v2:172.21.15.27:6808/2990360884,v1:172.21.15.27:6809/2990360884]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:47:21.091 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:47:21.091 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: pgmap v168: 97 pgs: 12 active+undersized+degraded, 19 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 46/435 objects degraded (10.575%) 2024-06-01T02:47:21.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='osd.5 ' entity='osd.5' 2024-06-01T02:47:21.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:21.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:21.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:47:21.092 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:47:21.093 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:21.093 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:21.093 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:47:21.093 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:21.093 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:21.093 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:21.094 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-01T02:47:21.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-01T02:47:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: osdmap e114: 8 total, 7 up, 8 in 2024-06-01T02:47:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='osd.5 [v2:172.21.15.27:6808/2990360884,v1:172.21.15.27:6809/2990360884]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:47:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:47:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: pgmap v168: 97 pgs: 12 active+undersized+degraded, 19 active+undersized, 66 active+clean; 240 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 46/435 objects degraded (10.575%) 2024-06-01T02:47:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='osd.5 ' entity='osd.5' 2024-06-01T02:47:21.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:21.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:21.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:47:21.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:47:21.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:21.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:21.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:47:21.105 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:21.105 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:21.105 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:21.105 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-01T02:47:22.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:21 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-01T02:47:22.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:21 smithi003 ceph-mon[67363]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-01T02:47:22.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:21 smithi003 ceph-mon[67363]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:47:22.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:21 smithi003 ceph-mon[67363]: osd.5 [v2:172.21.15.27:6808/2990360884,v1:172.21.15.27:6809/2990360884] boot 2024-06-01T02:47:22.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:21 smithi003 ceph-mon[67363]: osdmap e115: 8 total, 8 up, 8 in 2024-06-01T02:47:22.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:47:22.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:21 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-01T02:47:22.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:21 smithi027 ceph-mon[53801]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-01T02:47:22.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:21 smithi027 ceph-mon[53801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:47:22.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:21 smithi027 ceph-mon[53801]: osd.5 [v2:172.21.15.27:6808/2990360884,v1:172.21.15.27:6809/2990360884] boot 2024-06-01T02:47:22.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:21 smithi027 ceph-mon[53801]: osdmap e115: 8 total, 8 up, 8 in 2024-06-01T02:47:22.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-01T02:47:23.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:23 smithi003 ceph-mon[67363]: pgmap v170: 97 pgs: 7 peering, 7 active+undersized+degraded, 17 active+undersized, 66 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 24/435 objects degraded (5.517%) 2024-06-01T02:47:23.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:23 smithi003 ceph-mon[67363]: osdmap e116: 8 total, 8 up, 8 in 2024-06-01T02:47:23.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:23 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 24/435 objects degraded (5.517%), 7 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:23.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:23 smithi027 ceph-mon[53801]: pgmap v170: 97 pgs: 7 peering, 7 active+undersized+degraded, 17 active+undersized, 66 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 24/435 objects degraded (5.517%) 2024-06-01T02:47:23.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:23 smithi027 ceph-mon[53801]: osdmap e116: 8 total, 8 up, 8 in 2024-06-01T02:47:23.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:23 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 24/435 objects degraded (5.517%), 7 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:25.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:25 smithi003 ceph-mon[67363]: pgmap v172: 97 pgs: 7 peering, 5 active+undersized+degraded, 11 active+undersized, 74 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20/435 objects degraded (4.598%) 2024-06-01T02:47:25.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:25 smithi027 ceph-mon[53801]: pgmap v172: 97 pgs: 7 peering, 5 active+undersized+degraded, 11 active+undersized, 74 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20/435 objects degraded (4.598%) 2024-06-01T02:47:26.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:26 smithi003 ceph-mon[67363]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/435 objects degraded (4.598%), 5 pgs degraded) 2024-06-01T02:47:26.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:26 smithi003 ceph-mon[67363]: Cluster is now healthy 2024-06-01T02:47:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:26 smithi027 ceph-mon[53801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/435 objects degraded (4.598%), 5 pgs degraded) 2024-06-01T02:47:26.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:26 smithi027 ceph-mon[53801]: Cluster is now healthy 2024-06-01T02:47:27.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:27 smithi003 ceph-mon[67363]: pgmap v173: 97 pgs: 7 peering, 90 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-06-01T02:47:27.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:27 smithi027 ceph-mon[53801]: pgmap v173: 97 pgs: 7 peering, 90 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-06-01T02:47:29.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:29 smithi003 ceph-mon[67363]: pgmap v174: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:47:29.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:29 smithi027 ceph-mon[53801]: pgmap v174: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:47:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:31 smithi003 ceph-mon[67363]: pgmap v175: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 239 B/s rd, 0 op/s 2024-06-01T02:47:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:47:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:31 smithi027 ceph-mon[53801]: pgmap v175: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 239 B/s rd, 0 op/s 2024-06-01T02:47:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:47:33.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:33 smithi003 ceph-mon[67363]: pgmap v176: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:47:33.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:33 smithi027 ceph-mon[53801]: pgmap v176: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:47:35.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:35 smithi003 ceph-mon[67363]: pgmap v177: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 177 B/s rd, 0 op/s 2024-06-01T02:47:35.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:35 smithi027 ceph-mon[53801]: pgmap v177: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 177 B/s rd, 0 op/s 2024-06-01T02:47:36.539 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-01T02:47:36.539 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:36.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-01T02:47:36.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:37.322 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:37 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-01T02:47:37.323 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:37 smithi027 ceph-mon[53801]: Upgrade: osd.6 is safe to restart 2024-06-01T02:47:37.323 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:37 smithi027 ceph-mon[53801]: pgmap v178: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:37.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:37 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-01T02:47:37.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:37 smithi003 ceph-mon[67363]: Upgrade: osd.6 is safe to restart 2024-06-01T02:47:37.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:37 smithi003 ceph-mon[67363]: pgmap v178: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:37.690 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:47:38.374 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:47:38.374 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (6m) 91s ago 29m 13.7M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (3m) 91s ago 29m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (3m) 22s ago 29m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (6m) 91s ago 29m 44.8M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (23m) 91s ago 23m 40.7M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (23m) 22s ago 23m 18.9M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (5m) 91s ago 30m 508M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (5m) 22s ago 29m 453M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (5m) 91s ago 31m 50.2M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (4m) 22s ago 28m 40.1M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (23m) 91s ago 23m 295M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (23m) 22s ago 23m 88.5M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (6m) 91s ago 29m 8702k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:47:38.375 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (5m) 22s ago 28m 21.2M - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:47:38.376 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (3m) 91s ago 28m 194M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:47:38.376 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (2m) 91s ago 27m 175M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:47:38.376 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (2m) 91s ago 27m 147M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:47:38.376 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (98s) 91s ago 26m 15.0M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:47:38.376 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (61s) 22s ago 26m 200M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:47:38.376 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (26s) 22s ago 26m 11.5M 2704M 19.0.0-2946-g3d89accf b045ac545f82 cce965464b24 2024-06-01T02:47:38.376 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (25m) 22s ago 25m 375M 2704M 17.2.0 e1d6a67b021e 311ace34521d 2024-06-01T02:47:38.376 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (25m) 22s ago 25m 616M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:47:38.376 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (5m) 91s ago 28m 67.7M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:47:38.740 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:38 smithi027 ceph-mon[53801]: Upgrade: Updating osd.6 2024-06-01T02:47:38.740 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:38 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:38.741 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:38 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-01T02:47:38.741 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:38 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:47:38.741 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:38 smithi027 ceph-mon[53801]: Deploying daemon osd.6 on smithi027 2024-06-01T02:47:38.788 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:47:38.788 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:47:38.788 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 6 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-06-01T02:47:38.789 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 10 2024-06-01T02:47:38.790 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:47:38.790 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:47:38.790 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:38 smithi003 ceph-mon[67363]: Upgrade: Updating osd.6 2024-06-01T02:47:38.790 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:38 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:38.790 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:38 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-01T02:47:38.790 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:38 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:47:38.790 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:38 smithi003 ceph-mon[67363]: Deploying daemon osd.6 on smithi027 2024-06-01T02:47:39.138 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "12/23 daemons upgraded", 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading osd daemons", 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:47:39.139 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:47:39.552 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:47:39.611 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:39 smithi027 ceph-mon[53801]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:39.612 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:39 smithi027 ceph-mon[53801]: pgmap v179: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:47:39.612 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:39 smithi027 ceph-mon[53801]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:39.612 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:39 smithi027 ceph-mon[53801]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:39.612 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:39 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/2119669110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:39.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:39 smithi003 ceph-mon[67363]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:39.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:39 smithi003 ceph-mon[67363]: pgmap v179: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:47:39.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:39 smithi003 ceph-mon[67363]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:39.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:39 smithi003 ceph-mon[67363]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:39.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:39 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/2119669110' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:40.574 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:40 smithi027 ceph-mon[53801]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:40.574 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:40 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3602510884' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:47:40.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:40 smithi003 ceph-mon[67363]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:47:40.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:40 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3602510884' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:47:40.852 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:40 smithi027 systemd[1]: Stopping Ceph osd.6 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:47:41.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:41 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[37402]: 2024-06-01T02:47:41.027+0000 7fb324419700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:47:41.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:41 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[37402]: 2024-06-01T02:47:41.027+0000 7fb324419700 -1 osd.6 116 *** Got signal Terminated *** 2024-06-01T02:47:41.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:41 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[37402]: 2024-06-01T02:47:41.027+0000 7fb324419700 -1 osd.6 116 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:47:41.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:41 smithi003 ceph-mon[67363]: pgmap v180: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:41.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:41 smithi003 ceph-mon[67363]: osd.6 marked itself down and dead 2024-06-01T02:47:41.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:41 smithi027 ceph-mon[53801]: pgmap v180: 97 pgs: 97 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:47:41.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:41 smithi027 ceph-mon[53801]: osd.6 marked itself down and dead 2024-06-01T02:47:42.255 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:41 smithi027 podman[62673]: 2024-06-01 02:47:41.974218352 +0000 UTC m=+1.051439139 container died 311ace34521de37c54c30a6a03aec5d2bc702e648c0d33188a21fb3d70c4ea0d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, release=754, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-type=git, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True) 2024-06-01T02:47:42.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:42 smithi027 ceph-mon[53801]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:47:42.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:42 smithi027 ceph-mon[53801]: osdmap e117: 8 total, 7 up, 8 in 2024-06-01T02:47:42.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:42 smithi027 ceph-mon[53801]: osdmap e118: 8 total, 7 up, 8 in 2024-06-01T02:47:42.603 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:42 smithi027 podman[62673]: 2024-06-01 02:47:42.344963658 +0000 UTC m=+1.422184436 container cleanup 311ace34521de37c54c30a6a03aec5d2bc702e648c0d33188a21fb3d70c4ea0d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6, distribution-scope=public, vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, ceph=True, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8) 2024-06-01T02:47:42.603 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:42 smithi027 bash[62673]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6 2024-06-01T02:47:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:42 smithi003 ceph-mon[67363]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:47:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:42 smithi003 ceph-mon[67363]: osdmap e117: 8 total, 7 up, 8 in 2024-06-01T02:47:42.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:42 smithi003 ceph-mon[67363]: osdmap e118: 8 total, 7 up, 8 in 2024-06-01T02:47:42.990 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:42 smithi027 podman[62700]: 2024-06-01 02:47:42.621469584 +0000 UTC m=+0.640029341 container remove 311ace34521de37c54c30a6a03aec5d2bc702e648c0d33188a21fb3d70c4ea0d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, version=8, RELEASE=HEAD, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vcs-type=git, ceph=True, io.openshift.expose-services=, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD) 2024-06-01T02:47:43.241 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:42 smithi027 podman[62790]: 2024-06-01 02:47:42.893662357 +0000 UTC m=+0.018041565 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:47:43.242 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:43 smithi027 podman[62790]: 2024-06-01 02:47:43.005785374 +0000 UTC m=+0.130164566 container create 2314d363f1a69181e35228c4fa265bbc2cb409043731e641b478b6411f8c97f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-06-01T02:47:43.242 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:43 smithi027 podman[62790]: 2024-06-01 02:47:43.241886595 +0000 UTC m=+0.366265796 container init 2314d363f1a69181e35228c4fa265bbc2cb409043731e641b478b6411f8c97f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528) 2024-06-01T02:47:43.553 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:43 smithi027 podman[62790]: 2024-06-01 02:47:43.245486386 +0000 UTC m=+0.369865587 container start 2314d363f1a69181e35228c4fa265bbc2cb409043731e641b478b6411f8c97f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-06-01T02:47:43.553 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:43 smithi027 podman[62790]: 2024-06-01 02:47:43.350154174 +0000 UTC m=+0.474533387 container attach 2314d363f1a69181e35228c4fa265bbc2cb409043731e641b478b6411f8c97f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-3d89acc) 2024-06-01T02:47:43.553 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:43 smithi027 ceph-mon[53801]: pgmap v182: 97 pgs: 4 peering, 93 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:47:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:43 smithi003 ceph-mon[67363]: pgmap v182: 97 pgs: 4 peering, 93 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-01T02:47:43.852 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:43 smithi027 podman[62790]: 2024-06-01 02:47:43.55375023 +0000 UTC m=+0.678129436 container died 2314d363f1a69181e35228c4fa265bbc2cb409043731e641b478b6411f8c97f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:47:44.505 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 podman[62816]: 2024-06-01 02:47:44.111818559 +0000 UTC m=+0.551181780 container remove 2314d363f1a69181e35228c4fa265bbc2cb409043731e641b478b6411f8c97f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-06-01T02:47:44.506 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.6.service: Deactivated successfully. 2024-06-01T02:47:44.506 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 systemd[1]: Stopped Ceph osd.6 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:47:44.506 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.6.service: Consumed 3min 12.372s CPU time. 2024-06-01T02:47:44.506 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 systemd[1]: Starting Ceph osd.6 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:47:44.807 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:44 smithi027 ceph-mon[53801]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-06-01T02:47:44.807 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:44 smithi027 ceph-mon[53801]: Health check failed: Degraded data redundancy: 6/435 objects degraded (1.379%), 2 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:44.807 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 podman[62914]: 2024-06-01 02:47:44.471165791 +0000 UTC m=+0.018111953 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:47:44.808 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 podman[62914]: 2024-06-01 02:47:44.596712506 +0000 UTC m=+0.143658657 container create 1d09ae684b523bb5539b57b6e4b818a19260717f201e91d953fb048bfc0231cf (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS) 2024-06-01T02:47:44.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:44 smithi003 ceph-mon[67363]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-06-01T02:47:44.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:44 smithi003 ceph-mon[67363]: Health check failed: Degraded data redundancy: 6/435 objects degraded (1.379%), 2 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:45.102 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 podman[62914]: 2024-06-01 02:47:44.80778805 +0000 UTC m=+0.354734216 container init 1d09ae684b523bb5539b57b6e4b818a19260717f201e91d953fb048bfc0231cf (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate, GIT_CLEAN=True, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:47:45.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 podman[62914]: 2024-06-01 02:47:44.811442423 +0000 UTC m=+0.358388575 container start 1d09ae684b523bb5539b57b6e4b818a19260717f201e91d953fb048bfc0231cf (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-06-01T02:47:45.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:44 smithi027 podman[62914]: 2024-06-01 02:47:44.890970639 +0000 UTC m=+0.437916793 container attach 1d09ae684b523bb5539b57b6e4b818a19260717f201e91d953fb048bfc0231cf (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS) 2024-06-01T02:47:45.560 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:45 smithi027 ceph-mon[53801]: pgmap v184: 97 pgs: 7 stale+active+clean, 2 active+undersized+degraded, 12 peering, 2 active+undersized, 74 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 6/435 objects degraded (1.379%) 2024-06-01T02:47:45.561 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate[62929]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-01T02:47:45.561 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate[62929]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-01T02:47:45.561 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 bash[62914]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-01T02:47:45.561 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 bash[62914]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-01T02:47:45.561 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 bash[62914]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-01T02:47:45.561 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 bash[62914]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-01T02:47:45.561 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 bash[62914]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-06-01T02:47:45.562 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 bash[62914]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-01T02:47:45.562 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 bash[62914]: --> ceph-volume raw activate successful for osd ID: 6 2024-06-01T02:47:45.562 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate[62929]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-01T02:47:45.562 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate[62929]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-01T02:47:45.562 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate[62929]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-06-01T02:47:45.562 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate[62929]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-01T02:47:45.562 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate[62929]: --> ceph-volume raw activate successful for osd ID: 6 2024-06-01T02:47:45.562 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 podman[62914]: 2024-06-01 02:47:45.29859307 +0000 UTC m=+0.845539232 container died 1d09ae684b523bb5539b57b6e4b818a19260717f201e91d953fb048bfc0231cf (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate, RELEASE=squid-3d89acc, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True) 2024-06-01T02:47:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:45 smithi003 ceph-mon[67363]: pgmap v184: 97 pgs: 7 stale+active+clean, 2 active+undersized+degraded, 12 peering, 2 active+undersized, 74 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 6/435 objects degraded (1.379%) 2024-06-01T02:47:46.180 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:45 smithi027 podman[63045]: 2024-06-01 02:47:45.852952633 +0000 UTC m=+0.547957485 container remove 1d09ae684b523bb5539b57b6e4b818a19260717f201e91d953fb048bfc0231cf (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-activate, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-06-01T02:47:46.180 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:46 smithi027 podman[63103]: 2024-06-01 02:47:46.070344649 +0000 UTC m=+0.111403412 container create 13f0723f71f78d13f0923557c06c27e6d7ff7248f01f0f682f4accea622a50ea (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-06-01T02:47:46.180 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:46 smithi027 podman[63103]: 2024-06-01 02:47:45.977025164 +0000 UTC m=+0.018083940 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:47:46.602 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:46 smithi027 podman[63103]: 2024-06-01 02:47:46.256027013 +0000 UTC m=+0.297085783 container init 13f0723f71f78d13f0923557c06c27e6d7ff7248f01f0f682f4accea622a50ea (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:47:46.602 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:46 smithi027 podman[63103]: 2024-06-01 02:47:46.25937827 +0000 UTC m=+0.300437030 container start 13f0723f71f78d13f0923557c06c27e6d7ff7248f01f0f682f4accea622a50ea (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-3d89acc, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-06-01T02:47:46.603 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:46 smithi027 ceph-osd[63122]: -- 172.21.15.27:0/362960408 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557f2f1e5680 con 0x557f2f1aec00 2024-06-01T02:47:46.603 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:46 smithi027 bash[63103]: 13f0723f71f78d13f0923557c06c27e6d7ff7248f01f0f682f4accea622a50ea 2024-06-01T02:47:46.603 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:46 smithi027 systemd[1]: Started Ceph osd.6 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:47:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:46 smithi003 ceph-mon[67363]: pgmap v185: 97 pgs: 1 stale+active+clean, 8 active+undersized+degraded, 12 peering, 14 active+undersized, 62 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23/435 objects degraded (5.287%) 2024-06-01T02:47:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:47:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:47:47.102 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:46 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:47:46.803+0000 7ff5d82027c0 -1 Falling back to public interface 2024-06-01T02:47:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:46 smithi027 ceph-mon[53801]: pgmap v185: 97 pgs: 1 stale+active+clean, 8 active+undersized+degraded, 12 peering, 14 active+undersized, 62 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23/435 objects degraded (5.287%) 2024-06-01T02:47:47.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:47.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:47:47.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:47.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:47.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:47:48.222 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:47 smithi027 ceph-mon[53801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-06-01T02:47:48.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:47 smithi003 ceph-mon[67363]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-06-01T02:47:49.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:48 smithi003 ceph-mon[67363]: pgmap v186: 97 pgs: 15 active+undersized+degraded, 20 active+undersized, 62 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 52/435 objects degraded (11.954%) 2024-06-01T02:47:49.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:48 smithi027 ceph-mon[53801]: pgmap v186: 97 pgs: 15 active+undersized+degraded, 20 active+undersized, 62 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 52/435 objects degraded (11.954%) 2024-06-01T02:47:51.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:51 smithi003 ceph-mon[67363]: pgmap v187: 97 pgs: 15 active+undersized+degraded, 20 active+undersized, 62 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 52/435 objects degraded (11.954%) 2024-06-01T02:47:51.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:51 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:51.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:51 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:51.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:51 smithi027 ceph-mon[53801]: pgmap v187: 97 pgs: 15 active+undersized+degraded, 20 active+undersized, 62 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 52/435 objects degraded (11.954%) 2024-06-01T02:47:51.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:51 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:51.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:51 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:52.109 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:51 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:47:51.851+0000 7ff5d82027c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-06-01T02:47:52.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:52.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:52.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:52 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 52/435 objects degraded (11.954%), 15 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:52 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:52 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:52.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:52 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 52/435 objects degraded (11.954%), 15 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:52.983 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:47:52.729+0000 7ff5d82027c0 -1 osd.6 116 log_to_monitors true 2024-06-01T02:47:53.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:53 smithi003 ceph-mon[67363]: pgmap v188: 97 pgs: 15 active+undersized+degraded, 20 active+undersized, 62 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 52/435 objects degraded (11.954%) 2024-06-01T02:47:53.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:53 smithi003 ceph-mon[67363]: from='osd.6 [v2:172.21.15.27:6816/1115982981,v1:172.21.15.27:6817/1115982981]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-01T02:47:53.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:53 smithi003 ceph-mon[67363]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-01T02:47:53.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:53 smithi027 ceph-mon[53801]: pgmap v188: 97 pgs: 15 active+undersized+degraded, 20 active+undersized, 62 active+clean; 240 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 52/435 objects degraded (11.954%) 2024-06-01T02:47:53.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:53 smithi027 ceph-mon[53801]: from='osd.6 [v2:172.21.15.27:6816/1115982981,v1:172.21.15.27:6817/1115982981]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-01T02:47:53.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:53 smithi027 ceph-mon[53801]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-01T02:47:54.102 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:47:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:47:53.763+0000 7ff5c5651700 -1 osd.6 116 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:47:54.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:54 smithi003 ceph-mon[67363]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-01T02:47:54.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:54 smithi003 ceph-mon[67363]: osdmap e119: 8 total, 7 up, 8 in 2024-06-01T02:47:54.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:54 smithi003 ceph-mon[67363]: from='osd.6 [v2:172.21.15.27:6816/1115982981,v1:172.21.15.27:6817/1115982981]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:47:54.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:54 smithi003 ceph-mon[67363]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:47:54.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:54 smithi003 ceph-mon[67363]: from='osd.6 ' entity='osd.6' 2024-06-01T02:47:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:54 smithi027 ceph-mon[53801]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-01T02:47:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:54 smithi027 ceph-mon[53801]: osdmap e119: 8 total, 7 up, 8 in 2024-06-01T02:47:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:54 smithi027 ceph-mon[53801]: from='osd.6 [v2:172.21.15.27:6816/1115982981,v1:172.21.15.27:6817/1115982981]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:47:54.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:54 smithi027 ceph-mon[53801]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:47:54.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:54 smithi027 ceph-mon[53801]: from='osd.6 ' entity='osd.6' 2024-06-01T02:47:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:55 smithi003 ceph-mon[67363]: pgmap v190: 97 pgs: 15 active+undersized+degraded, 20 active+undersized, 62 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 52/435 objects degraded (11.954%) 2024-06-01T02:47:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:55 smithi003 ceph-mon[67363]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:47:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:55 smithi003 ceph-mon[67363]: osd.6 [v2:172.21.15.27:6816/1115982981,v1:172.21.15.27:6817/1115982981] boot 2024-06-01T02:47:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:55 smithi003 ceph-mon[67363]: osdmap e120: 8 total, 8 up, 8 in 2024-06-01T02:47:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:47:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:55 smithi027 ceph-mon[53801]: pgmap v190: 97 pgs: 15 active+undersized+degraded, 20 active+undersized, 62 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 52/435 objects degraded (11.954%) 2024-06-01T02:47:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:55 smithi027 ceph-mon[53801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:47:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:55 smithi027 ceph-mon[53801]: osd.6 [v2:172.21.15.27:6816/1115982981,v1:172.21.15.27:6817/1115982981] boot 2024-06-01T02:47:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:55 smithi027 ceph-mon[53801]: osdmap e120: 8 total, 8 up, 8 in 2024-06-01T02:47:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-01T02:47:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:47:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:47:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:56.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:47:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-01T02:47:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-01T02:47:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-01T02:47:56.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:56 smithi003 ceph-mon[67363]: osdmap e121: 8 total, 8 up, 8 in 2024-06-01T02:47:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-01T02:47:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:56 smithi027 ceph-mon[53801]: osdmap e121: 8 total, 8 up, 8 in 2024-06-01T02:47:57.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:57 smithi003 ceph-mon[67363]: pgmap v193: 97 pgs: 12 peering, 13 active+undersized+degraded, 10 active+undersized, 62 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 46/435 objects degraded (10.575%) 2024-06-01T02:47:57.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:57 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 46/435 objects degraded (10.575%), 13 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:57 smithi027 ceph-mon[53801]: pgmap v193: 97 pgs: 12 peering, 13 active+undersized+degraded, 10 active+undersized, 62 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 46/435 objects degraded (10.575%) 2024-06-01T02:47:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:57 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 46/435 objects degraded (10.575%), 13 pgs degraded (PG_DEGRADED) 2024-06-01T02:47:59.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:47:59 smithi003 ceph-mon[67363]: pgmap v194: 97 pgs: 12 peering, 9 active+undersized+degraded, 4 active+undersized, 72 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 30/435 objects degraded (6.897%) 2024-06-01T02:47:59.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:47:59 smithi027 ceph-mon[53801]: pgmap v194: 97 pgs: 12 peering, 9 active+undersized+degraded, 4 active+undersized, 72 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 30/435 objects degraded (6.897%) 2024-06-01T02:48:00.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:00 smithi003 ceph-mon[67363]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/435 objects degraded (6.897%), 9 pgs degraded) 2024-06-01T02:48:00.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:00 smithi003 ceph-mon[67363]: Cluster is now healthy 2024-06-01T02:48:00.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:00 smithi027 ceph-mon[53801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/435 objects degraded (6.897%), 9 pgs degraded) 2024-06-01T02:48:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:00 smithi027 ceph-mon[53801]: Cluster is now healthy 2024-06-01T02:48:01.738 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:01 smithi003 ceph-mon[67363]: pgmap v195: 97 pgs: 7 peering, 90 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 314 B/s rd, 0 op/s 2024-06-01T02:48:01.738 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:01.738 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:48:01.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:01 smithi027 ceph-mon[53801]: pgmap v195: 97 pgs: 7 peering, 90 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 314 B/s rd, 0 op/s 2024-06-01T02:48:01.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:01.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:48:03.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:03 smithi003 ceph-mon[67363]: pgmap v196: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:48:03.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:03 smithi027 ceph-mon[53801]: pgmap v196: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:48:05.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:05 smithi003 ceph-mon[67363]: pgmap v197: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 226 B/s rd, 0 op/s 2024-06-01T02:48:05.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:05 smithi027 ceph-mon[53801]: pgmap v197: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 226 B/s rd, 0 op/s 2024-06-01T02:48:07.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:07 smithi003 ceph-mon[67363]: pgmap v198: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-06-01T02:48:07.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:07 smithi027 ceph-mon[53801]: pgmap v198: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-06-01T02:48:09.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:09 smithi003 ceph-mon[67363]: pgmap v199: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:48:09.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:09 smithi027 ceph-mon[53801]: pgmap v199: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:48:09.932 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:48:10.616 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:48:10.616 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (6m) 2m ago 30m 13.7M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (4m) 2m ago 30m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (4m) 20s ago 29m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (6m) 2m ago 29m 44.8M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (24m) 2m ago 24m 40.7M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (24m) 20s ago 24m 19.0M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (6m) 2m ago 31m 508M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (5m) 20s ago 29m 453M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (5m) 2m ago 31m 50.2M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (5m) 20s ago 29m 41.5M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (24m) 2m ago 24m 295M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (23m) 20s ago 23m 90.6M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:48:10.617 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (6m) 2m ago 29m 8702k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (6m) 20s ago 29m 18.8M - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (4m) 2m ago 28m 194M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (3m) 2m ago 28m 175M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (2m) 2m ago 27m 147M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (2m) 2m ago 27m 15.0M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (93s) 20s ago 27m 201M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (59s) 20s ago 26m 190M 2704M 19.0.0-2946-g3d89accf b045ac545f82 cce965464b24 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (24s) 20s ago 26m 14.8M 2704M 19.0.0-2946-g3d89accf b045ac545f82 13f0723f71f7 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (25m) 20s ago 25m 615M 2704M 17.2.0 e1d6a67b021e 62d9a8fe3289 2024-06-01T02:48:10.618 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (5m) 2m ago 29m 67.7M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:48:11.033 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:48:11.033 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:48:11.033 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:48:11.033 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:48:11.033 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:48:11.033 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:48:11.033 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 7 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 11 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:48:11.034 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:48:11.380 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:48:11.380 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:48:11.380 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:48:11.380 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:48:11.381 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:48:11.381 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:48:11.381 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:48:11.381 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:48:11.381 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:48:11.381 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "13/23 daemons upgraded", 2024-06-01T02:48:11.381 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading osd daemons", 2024-06-01T02:48:11.381 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:48:11.381 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:48:11.750 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:11 smithi027 ceph-mon[53801]: pgmap v200: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:48:11.750 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:11 smithi027 ceph-mon[53801]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:11.750 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:11 smithi027 ceph-mon[53801]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:11.750 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-01T02:48:11.751 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:11 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-01T02:48:11.751 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:11 smithi027 ceph-mon[53801]: Upgrade: osd.7 is safe to restart 2024-06-01T02:48:11.751 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:11.751 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:11 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/4244354711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:11.796 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:48:11.797 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:11 smithi003 ceph-mon[67363]: pgmap v200: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:48:11.797 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:11 smithi003 ceph-mon[67363]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:11.797 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:11 smithi003 ceph-mon[67363]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:11.797 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-01T02:48:11.797 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:11 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-01T02:48:11.797 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:11 smithi003 ceph-mon[67363]: Upgrade: osd.7 is safe to restart 2024-06-01T02:48:11.797 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:11.797 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:11 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/4244354711' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:12.775 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:12 smithi027 ceph-mon[53801]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:12.776 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:12 smithi027 ceph-mon[53801]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:12.776 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:12 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/252514202' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:48:12.776 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:12.776 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-01T02:48:12.776 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:12 smithi003 ceph-mon[67363]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:12 smithi003 ceph-mon[67363]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:12 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/252514202' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:48:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-01T02:48:12.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:13.682 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:13 smithi027 ceph-mon[53801]: pgmap v201: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:48:13.682 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:13 smithi027 ceph-mon[53801]: Upgrade: Updating osd.7 2024-06-01T02:48:13.682 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:13 smithi027 ceph-mon[53801]: Deploying daemon osd.7 on smithi027 2024-06-01T02:48:13.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:13 smithi003 ceph-mon[67363]: pgmap v201: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:48:13.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:13 smithi003 ceph-mon[67363]: Upgrade: Updating osd.7 2024-06-01T02:48:13.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:13 smithi003 ceph-mon[67363]: Deploying daemon osd.7 on smithi027 2024-06-01T02:48:15.645 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:15 smithi027 ceph-mon[53801]: pgmap v202: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:48:15.645 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:15 smithi027 systemd[1]: Stopping Ceph osd.7 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:48:15.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:15 smithi003 ceph-mon[67363]: pgmap v202: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:48:16.102 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:15 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[40459]: 2024-06-01T02:48:15.652+0000 7fd5599f9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:48:16.102 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:15 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[40459]: 2024-06-01T02:48:15.652+0000 7fd5599f9700 -1 osd.7 121 *** Got signal Terminated *** 2024-06-01T02:48:16.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:15 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[40459]: 2024-06-01T02:48:15.652+0000 7fd5599f9700 -1 osd.7 121 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:48:16.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:16 smithi003 ceph-mon[67363]: osd.7 marked itself down and dead 2024-06-01T02:48:16.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:48:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:16 smithi027 ceph-mon[53801]: osd.7 marked itself down and dead 2024-06-01T02:48:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:48:17.211 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:16 smithi027 podman[64909]: 2024-06-01 02:48:16.919503994 +0000 UTC m=+1.374325559 container died 62d9a8fe3289de3bc23a7cf4629967a6b476b28acc685394e193dec4277aa805 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, RELEASE=HEAD, architecture=x86_64, com.redhat.component=centos-stream-container, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8) 2024-06-01T02:48:17.544 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:17 smithi027 ceph-mon[53801]: pgmap v203: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:48:17.544 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:17 smithi027 ceph-mon[53801]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:48:17.544 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:17 smithi027 ceph-mon[53801]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-01T02:48:17.544 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:17 smithi027 ceph-mon[53801]: osdmap e122: 8 total, 7 up, 8 in 2024-06-01T02:48:17.545 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:17 smithi027 podman[64909]: 2024-06-01 02:48:17.302016326 +0000 UTC m=+1.756837890 container cleanup 62d9a8fe3289de3bc23a7cf4629967a6b476b28acc685394e193dec4277aa805 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vcs-type=git, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, architecture=x86_64, io.openshift.expose-services=, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_CLEAN=True, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0) 2024-06-01T02:48:17.545 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:17 smithi027 bash[64909]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7 2024-06-01T02:48:17.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:17 smithi003 ceph-mon[67363]: pgmap v203: 97 pgs: 97 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:48:17.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:17 smithi003 ceph-mon[67363]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-01T02:48:17.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:17 smithi003 ceph-mon[67363]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-01T02:48:17.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:17 smithi003 ceph-mon[67363]: osdmap e122: 8 total, 7 up, 8 in 2024-06-01T02:48:17.852 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:17 smithi027 podman[64956]: 2024-06-01 02:48:17.544796832 +0000 UTC m=+0.619576284 container remove 62d9a8fe3289de3bc23a7cf4629967a6b476b28acc685394e193dec4277aa805 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7, version=8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, architecture=x86_64, GIT_CLEAN=True, ceph=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., release=754, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870) 2024-06-01T02:48:18.181 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:17 smithi027 podman[65046]: 2024-06-01 02:48:17.803670727 +0000 UTC m=+0.018423766 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:48:18.182 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:17 smithi027 podman[65046]: 2024-06-01 02:48:17.904116885 +0000 UTC m=+0.118869908 container create 31113d01aedc28ee1bac6c420fe45251570525c9c073999f284a856813331ee6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, GIT_CLEAN=True, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:48:18.182 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:18 smithi027 podman[65046]: 2024-06-01 02:48:18.182139771 +0000 UTC m=+0.396892803 container init 31113d01aedc28ee1bac6c420fe45251570525c9c073999f284a856813331ee6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-06-01T02:48:18.494 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:18 smithi027 ceph-mon[53801]: osdmap e123: 8 total, 7 up, 8 in 2024-06-01T02:48:18.495 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:18 smithi027 podman[65046]: 2024-06-01 02:48:18.185768152 +0000 UTC m=+0.400521176 container start 31113d01aedc28ee1bac6c420fe45251570525c9c073999f284a856813331ee6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD) 2024-06-01T02:48:18.495 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:18 smithi027 podman[65046]: 2024-06-01 02:48:18.290308638 +0000 UTC m=+0.505061676 container attach 31113d01aedc28ee1bac6c420fe45251570525c9c073999f284a856813331ee6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, ceph=True, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-06-01T02:48:18.793 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:18 smithi027 podman[65046]: 2024-06-01 02:48:18.495207167 +0000 UTC m=+0.709960193 container died 31113d01aedc28ee1bac6c420fe45251570525c9c073999f284a856813331ee6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-06-01T02:48:18.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:18 smithi003 ceph-mon[67363]: osdmap e123: 8 total, 7 up, 8 in 2024-06-01T02:48:19.344 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 podman[65072]: 2024-06-01 02:48:19.094087344 +0000 UTC m=+0.588337037 container remove 31113d01aedc28ee1bac6c420fe45251570525c9c073999f284a856813331ee6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-06-01T02:48:19.344 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.7.service: Deactivated successfully. 2024-06-01T02:48:19.344 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 systemd[1]: Stopped Ceph osd.7 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:48:19.344 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 systemd[1]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.7.service: Consumed 4min 34.623s CPU time. 2024-06-01T02:48:19.344 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 systemd[1]: Starting Ceph osd.7 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:48:19.678 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:19 smithi027 ceph-mon[53801]: pgmap v206: 97 pgs: 17 peering, 80 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-06-01T02:48:19.679 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:19 smithi027 ceph-mon[53801]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-06-01T02:48:19.679 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 podman[65170]: 2024-06-01 02:48:19.467235856 +0000 UTC m=+0.017956351 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:48:19.679 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 podman[65170]: 2024-06-01 02:48:19.570168336 +0000 UTC m=+0.120888823 container create a820b163d8974b4030f69819ac769a609b768be40c6baa15648b1c156ffab6fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS) 2024-06-01T02:48:19.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:19 smithi003 ceph-mon[67363]: pgmap v206: 97 pgs: 17 peering, 80 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-06-01T02:48:19.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:19 smithi003 ceph-mon[67363]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-06-01T02:48:20.102 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 podman[65170]: 2024-06-01 02:48:19.781368583 +0000 UTC m=+0.332089071 container init a820b163d8974b4030f69819ac769a609b768be40c6baa15648b1c156ffab6fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-3d89acc) 2024-06-01T02:48:20.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 podman[65170]: 2024-06-01 02:48:19.785186509 +0000 UTC m=+0.335906996 container start a820b163d8974b4030f69819ac769a609b768be40c6baa15648b1c156ffab6fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528) 2024-06-01T02:48:20.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:19 smithi027 podman[65170]: 2024-06-01 02:48:19.881344646 +0000 UTC m=+0.432065129 container attach a820b163d8974b4030f69819ac769a609b768be40c6baa15648b1c156ffab6fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-06-01T02:48:20.593 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:20 smithi027 ceph-mon[53801]: Health check failed: Degraded data redundancy: 48/435 objects degraded (11.034%), 12 pgs degraded (PG_DEGRADED) 2024-06-01T02:48:20.593 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate[65185]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate[65185]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 bash[65170]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 bash[65170]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 bash[65170]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 bash[65170]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 bash[65170]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 bash[65170]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 bash[65170]: --> ceph-volume raw activate successful for osd ID: 7 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate[65185]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate[65185]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate[65185]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate[65185]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-01T02:48:20.594 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate[65185]: --> ceph-volume raw activate successful for osd ID: 7 2024-06-01T02:48:20.595 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 podman[65170]: 2024-06-01 02:48:20.279621487 +0000 UTC m=+0.830341975 container died a820b163d8974b4030f69819ac769a609b768be40c6baa15648b1c156ffab6fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True) 2024-06-01T02:48:20.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:20 smithi003 ceph-mon[67363]: Health check failed: Degraded data redundancy: 48/435 objects degraded (11.034%), 12 pgs degraded (PG_DEGRADED) 2024-06-01T02:48:21.135 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:20 smithi027 podman[65311]: 2024-06-01 02:48:20.885204575 +0000 UTC m=+0.598582537 container remove a820b163d8974b4030f69819ac769a609b768be40c6baa15648b1c156ffab6fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-06-01T02:48:21.135 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:21 smithi027 podman[65373]: 2024-06-01 02:48:21.006390878 +0000 UTC m=+0.017949654 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:48:21.136 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:21 smithi027 podman[65373]: 2024-06-01 02:48:21.136078591 +0000 UTC m=+0.147637367 container create 196fb45013fd362c862a492923b547fc9d737eb8c78f0946d85df1f7d3c9c039 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-06-01T02:48:21.390 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:21 smithi027 podman[65373]: 2024-06-01 02:48:21.363466076 +0000 UTC m=+0.375024843 container init 196fb45013fd362c862a492923b547fc9d737eb8c78f0946d85df1f7d3c9c039 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:48:21.390 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:21 smithi027 podman[65373]: 2024-06-01 02:48:21.366887644 +0000 UTC m=+0.378446420 container start 196fb45013fd362c862a492923b547fc9d737eb8c78f0946d85df1f7d3c9c039 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240528, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS) 2024-06-01T02:48:21.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:21 smithi003 ceph-mon[67363]: pgmap v207: 97 pgs: 12 active+undersized+degraded, 17 peering, 5 active+undersized, 63 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:48:21.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:21 smithi027 ceph-mon[53801]: pgmap v207: 97 pgs: 12 active+undersized+degraded, 17 peering, 5 active+undersized, 63 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 48/435 objects degraded (11.034%) 2024-06-01T02:48:21.852 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:21 smithi027 ceph-osd[65393]: -- 172.21.15.27:0/3249663265 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5604fa5e3680 con 0x5604fa5ad000 2024-06-01T02:48:21.852 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:21 smithi027 bash[65373]: 196fb45013fd362c862a492923b547fc9d737eb8c78f0946d85df1f7d3c9c039 2024-06-01T02:48:21.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:21 smithi027 systemd[1]: Started Ceph osd.7 for e42641dc-1fbc-11ef-bc9b-c7b262605968. 2024-06-01T02:48:22.319 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:21 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:48:21.914+0000 7f7d174a07c0 -1 Falling back to public interface 2024-06-01T02:48:22.574 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:22.574 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:22.574 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:22 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:48:22.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:22.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:22.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:22 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:48:23.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:23 smithi027 ceph-mon[53801]: pgmap v208: 97 pgs: 14 active+undersized+degraded, 17 peering, 11 active+undersized, 55 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 55/435 objects degraded (12.644%) 2024-06-01T02:48:23.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:23 smithi003 ceph-mon[67363]: pgmap v208: 97 pgs: 14 active+undersized+degraded, 17 peering, 11 active+undersized, 55 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 55/435 objects degraded (12.644%) 2024-06-01T02:48:24.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:24 smithi003 ceph-mon[67363]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2024-06-01T02:48:24.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:24 smithi027 ceph-mon[53801]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2024-06-01T02:48:25.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:25 smithi003 ceph-mon[67363]: pgmap v209: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70/435 objects degraded (16.092%) 2024-06-01T02:48:25.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:25.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:25.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:25 smithi027 ceph-mon[53801]: pgmap v209: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70/435 objects degraded (16.092%) 2024-06-01T02:48:25.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:25.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:27 smithi003 ceph-mon[67363]: pgmap v210: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70/435 objects degraded (16.092%) 2024-06-01T02:48:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:27 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:27 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:27 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 70/435 objects degraded (16.092%), 21 pgs degraded (PG_DEGRADED) 2024-06-01T02:48:27.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:27 smithi027 ceph-mon[53801]: pgmap v210: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70/435 objects degraded (16.092%) 2024-06-01T02:48:27.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:27 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:27.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:27 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:27.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:27 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 70/435 objects degraded (16.092%), 21 pgs degraded (PG_DEGRADED) 2024-06-01T02:48:28.546 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:28 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:48:28.274+0000 7f7d174a07c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-06-01T02:48:29.761 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:29 smithi027 ceph-mon[53801]: pgmap v211: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 196 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:48:29.762 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:29 smithi027 ceph-mon[53801]: from='osd.7 [v2:172.21.15.27:6824/2236837890,v1:172.21.15.27:6825/2236837890]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-01T02:48:29.762 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:29 smithi027 ceph-mon[53801]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-01T02:48:29.762 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:29 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:48:29.557+0000 7f7d174a07c0 -1 osd.7 121 log_to_monitors true 2024-06-01T02:48:29.762 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:48:29 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:48:29.606+0000 7f7d088f7700 -1 osd.7 121 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-01T02:48:29.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:29 smithi003 ceph-mon[67363]: pgmap v211: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 196 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:48:29.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:29 smithi003 ceph-mon[67363]: from='osd.7 [v2:172.21.15.27:6824/2236837890,v1:172.21.15.27:6825/2236837890]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-01T02:48:29.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:29 smithi003 ceph-mon[67363]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-01T02:48:30.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-01T02:48:30.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='osd.7 [v2:172.21.15.27:6824/2236837890,v1:172.21.15.27:6825/2236837890]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:48:30.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: osdmap e124: 8 total, 7 up, 8 in 2024-06-01T02:48:30.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:48:30.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: pgmap v213: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:48:30.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:30.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all osd 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:30.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-01T02:48:30.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-01T02:48:30.856 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-01T02:48:30.856 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-01T02:48:30.856 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-01T02:48:30.856 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: Upgrade: Setting require_osd_release to 19 squid 2024-06-01T02:48:30.856 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-01T02:48:31.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-01T02:48:31.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='osd.7 [v2:172.21.15.27:6824/2236837890,v1:172.21.15.27:6825/2236837890]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:48:31.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: osdmap e124: 8 total, 7 up, 8 in 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: pgmap v213: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:31.088 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all osd 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-01T02:48:31.089 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-01T02:48:31.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-01T02:48:31.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-01T02:48:31.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-01T02:48:31.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-01T02:48:31.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: Upgrade: Setting require_osd_release to 19 squid 2024-06-01T02:48:31.090 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: osd.7 [v2:172.21.15.27:6824/2236837890,v1:172.21.15.27:6825/2236837890] boot 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: osdmap e125: 8 total, 8 up, 8 in 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi003.kexxec"]}]: dispatch 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: Upgrade: It appears safe to stop mds.foofs.smithi003.kexxec 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:48:31.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:31 smithi003 ceph-mon[67363]: osdmap e126: 8 total, 8 up, 8 in 2024-06-01T02:48:31.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: osd.7 [v2:172.21.15.27:6824/2236837890,v1:172.21.15.27:6825/2236837890] boot 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: osdmap e125: 8 total, 8 up, 8 in 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi003.kexxec"]}]: dispatch 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: Upgrade: It appears safe to stop mds.foofs.smithi003.kexxec 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:48:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:31 smithi027 ceph-mon[53801]: osdmap e126: 8 total, 8 up, 8 in 2024-06-01T02:48:33.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:33 smithi003 ceph-mon[67363]: pgmap v216: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:48:33.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:33 smithi003 ceph-mon[67363]: Upgrade: Updating mds.foofs.smithi003.kexxec 2024-06-01T02:48:33.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:33 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:33.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:33 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi003.kexxec", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:48:33.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:33 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:33.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:33 smithi003 ceph-mon[67363]: Deploying daemon mds.foofs.smithi003.kexxec on smithi003 2024-06-01T02:48:33.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:33 smithi027 ceph-mon[53801]: pgmap v216: 97 pgs: 21 active+undersized+degraded, 21 active+undersized, 55 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 70/435 objects degraded (16.092%) 2024-06-01T02:48:33.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:33 smithi027 ceph-mon[53801]: Upgrade: Updating mds.foofs.smithi003.kexxec 2024-06-01T02:48:33.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:33 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:33.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:33 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi003.kexxec", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:48:33.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:33 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:33.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:33 smithi027 ceph-mon[53801]: Deploying daemon mds.foofs.smithi003.kexxec on smithi003 2024-06-01T02:48:34.631 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:34 smithi003 ceph-mon[67363]: Health check update: Degraded data redundancy: 39/435 objects degraded (8.966%), 10 pgs degraded (PG_DEGRADED) 2024-06-01T02:48:34.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:34 smithi027 ceph-mon[53801]: Health check update: Degraded data redundancy: 39/435 objects degraded (8.966%), 10 pgs degraded (PG_DEGRADED) 2024-06-01T02:48:35.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:34 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003[67359]: 2024-06-01T02:48:34.631+0000 7fa4152df700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-01T02:48:35.087 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:48:34 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:48:34.655+0000 7f63a9d06700 -1 bdev(0x55a6587b6000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 7 2024-06-01T02:48:35.087 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:48:34 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:48:34.656+0000 7f63a4cfc700 -1 bdev(0x55a6587b6000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 4 2024-06-01T02:48:35.087 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:48:34 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:48:34.657+0000 7f63ab509700 -1 bdev(0x55a6587b6000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 7 2024-06-01T02:48:35.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:35 smithi003 ceph-mon[67363]: pgmap v217: 97 pgs: 10 active+undersized+degraded, 12 active+undersized, 75 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 39/435 objects degraded (8.966%) 2024-06-01T02:48:35.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:35 smithi003 ceph-mon[67363]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-06-01T02:48:35.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:35 smithi003 ceph-mon[67363]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-01T02:48:35.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:35 smithi003 ceph-mon[67363]: osdmap e127: 8 total, 8 up, 8 in 2024-06-01T02:48:35.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:35 smithi003 ceph-mon[67363]: Standby daemon mds.foofs.smithi027.xaykap assigned to filesystem foofs as rank 0 2024-06-01T02:48:35.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:35 smithi003 ceph-mon[67363]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-06-01T02:48:35.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:35 smithi003 ceph-mon[67363]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-01T02:48:35.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:35 smithi003 ceph-mon[67363]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-06-01T02:48:35.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:35 smithi003 ceph-mon[67363]: fsmap foofs:1/1 {0=foofs.smithi027.xaykap=up:replay} 2024-06-01T02:48:35.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:35 smithi027 ceph-mon[53801]: pgmap v217: 97 pgs: 10 active+undersized+degraded, 12 active+undersized, 75 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 39/435 objects degraded (8.966%) 2024-06-01T02:48:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:35 smithi027 ceph-mon[53801]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-06-01T02:48:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:35 smithi027 ceph-mon[53801]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-01T02:48:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:35 smithi027 ceph-mon[53801]: osdmap e127: 8 total, 8 up, 8 in 2024-06-01T02:48:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:35 smithi027 ceph-mon[53801]: Standby daemon mds.foofs.smithi027.xaykap assigned to filesystem foofs as rank 0 2024-06-01T02:48:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:35 smithi027 ceph-mon[53801]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-06-01T02:48:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:35 smithi027 ceph-mon[53801]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-01T02:48:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:35 smithi027 ceph-mon[53801]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-06-01T02:48:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:35 smithi027 ceph-mon[53801]: fsmap foofs:1/1 {0=foofs.smithi027.xaykap=up:replay} 2024-06-01T02:48:36.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:36 smithi003 ceph-mon[67363]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 39/435 objects degraded (8.966%), 10 pgs degraded) 2024-06-01T02:48:36.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:36 smithi003 ceph-mon[67363]: mds.? [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] up:reconnect 2024-06-01T02:48:36.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:36 smithi003 ceph-mon[67363]: fsmap foofs:1/1 {0=foofs.smithi027.xaykap=up:reconnect} 2024-06-01T02:48:36.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:36 smithi027 ceph-mon[53801]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 39/435 objects degraded (8.966%), 10 pgs degraded) 2024-06-01T02:48:36.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:36 smithi027 ceph-mon[53801]: mds.? [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] up:reconnect 2024-06-01T02:48:36.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:36 smithi027 ceph-mon[53801]: fsmap foofs:1/1 {0=foofs.smithi027.xaykap=up:reconnect} 2024-06-01T02:48:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:37 smithi003 ceph-mon[67363]: pgmap v219: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 0 op/s 2024-06-01T02:48:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:37 smithi003 ceph-mon[67363]: reconnect by client.24357 172.21.15.27:0/1146346812 after 0 2024-06-01T02:48:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:37 smithi003 ceph-mon[67363]: reconnect by client.14562 172.21.15.3:0/3874295321 after 0.000999973 2024-06-01T02:48:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:37 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:48:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:37 smithi003 ceph-mon[67363]: mds.? [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] up:rejoin 2024-06-01T02:48:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:37 smithi003 ceph-mon[67363]: fsmap foofs:1/1 {0=foofs.smithi027.xaykap=up:rejoin} 2024-06-01T02:48:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:37 smithi003 ceph-mon[67363]: daemon mds.foofs.smithi027.xaykap is now active in filesystem foofs as rank 0 2024-06-01T02:48:37.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:37 smithi027 ceph-mon[53801]: pgmap v219: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 0 op/s 2024-06-01T02:48:37.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:37 smithi027 ceph-mon[53801]: reconnect by client.24357 172.21.15.27:0/1146346812 after 0 2024-06-01T02:48:37.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:37 smithi027 ceph-mon[53801]: reconnect by client.14562 172.21.15.3:0/3874295321 after 0.000999973 2024-06-01T02:48:37.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:37 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:48:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:37 smithi027 ceph-mon[53801]: mds.? [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] up:rejoin 2024-06-01T02:48:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:37 smithi027 ceph-mon[53801]: fsmap foofs:1/1 {0=foofs.smithi027.xaykap=up:rejoin} 2024-06-01T02:48:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:37 smithi027 ceph-mon[53801]: daemon mds.foofs.smithi027.xaykap is now active in filesystem foofs as rank 0 2024-06-01T02:48:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:38 smithi003 ceph-mon[67363]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-06-01T02:48:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:38 smithi003 ceph-mon[67363]: mds.? [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] up:active 2024-06-01T02:48:38.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:38 smithi003 ceph-mon[67363]: fsmap foofs:1 {0=foofs.smithi027.xaykap=up:active} 2024-06-01T02:48:38.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:38 smithi027 ceph-mon[53801]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-06-01T02:48:38.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:38 smithi027 ceph-mon[53801]: mds.? [v2:172.21.15.27:6832/3085051232,v1:172.21.15.27:6833/3085051232] up:active 2024-06-01T02:48:38.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:38 smithi027 ceph-mon[53801]: fsmap foofs:1 {0=foofs.smithi027.xaykap=up:active} 2024-06-01T02:48:39.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:39 smithi003 ceph-mon[67363]: pgmap v220: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 2 op/s 2024-06-01T02:48:39.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:39 smithi027 ceph-mon[53801]: pgmap v220: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 2 op/s 2024-06-01T02:48:41.745 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:41 smithi027 ceph-mon[53801]: pgmap v221: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 248 B/s wr, 12 op/s 2024-06-01T02:48:41.746 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:41 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:41.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:41 smithi003 ceph-mon[67363]: pgmap v221: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 248 B/s wr, 12 op/s 2024-06-01T02:48:41.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:41 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:42.175 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (7m) 2m ago 30m 13.7M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (5m) 2m ago 30m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (4m) 17s ago 30m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (7m) 2m ago 30m 44.8M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (24m) 2m ago 24m 40.7M - 17.2.0 e1d6a67b021e 4c2ed0e67e37 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (24m) 17s ago 24m 19.1M - 17.2.0 e1d6a67b021e bd0e059f0cbe 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (6m) 2m ago 31m 508M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (6m) 17s ago 30m 454M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (6m) 2m ago 32m 50.2M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (5m) 17s ago 30m 42.4M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (24m) 2m ago 24m 295M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:48:42.864 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (24m) 17s ago 24m 90.7M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (7m) 2m ago 30m 8702k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (7m) 17s ago 29m 18.5M - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (4m) 2m ago 29m 194M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (3m) 2m ago 28m 175M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (3m) 2m ago 28m 147M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (2m) 2m ago 27m 15.0M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (2m) 17s ago 27m 202M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (91s) 17s ago 27m 191M 2704M 19.0.0-2946-g3d89accf b045ac545f82 cce965464b24 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (56s) 17s ago 26m 180M 2704M 19.0.0-2946-g3d89accf b045ac545f82 13f0723f71f7 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (21s) 17s ago 26m 14.9M 2704M 19.0.0-2946-g3d89accf b045ac545f82 196fb45013fd 2024-06-01T02:48:42.865 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (6m) 2m ago 29m 67.7M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:48:43.279 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:48:43.279 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 8 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:48:43.280 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-01T02:48:43.281 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 12 2024-06-01T02:48:43.281 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:48:43.281 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:48:43.639 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr", 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "osd" 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "14/23 daemons upgraded", 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading mds daemons", 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:48:43.640 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:48:43.795 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:43 smithi003 ceph-mon[67363]: pgmap v222: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 9 op/s 2024-06-01T02:48:43.796 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:43 smithi003 ceph-mon[67363]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:43.796 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:43 smithi003 ceph-mon[67363]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:43.796 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:43 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/4263222920' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:43.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:43 smithi027 ceph-mon[53801]: pgmap v222: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 9 op/s 2024-06-01T02:48:43.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:43 smithi027 ceph-mon[53801]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:43.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:43 smithi027 ceph-mon[53801]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:43.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:43 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/4263222920' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:44.091 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_WARN insufficient standby MDS daemons available 2024-06-01T02:48:44.092 INFO:teuthology.orchestra.run.smithi003.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-06-01T02:48:44.092 INFO:teuthology.orchestra.run.smithi003.stdout: have 0; want 1 more 2024-06-01T02:48:44.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:44 smithi003 ceph-mon[67363]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:44.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:44 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1239881970' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:48:44.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:44 smithi027 ceph-mon[53801]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:44.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:44 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/1239881970' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:48:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: pgmap v223: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 11 op/s 2024-06-01T02:48:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:48:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-06-01T02:48:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: Cluster is now healthy 2024-06-01T02:48:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670] up:boot 2024-06-01T02:48:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: fsmap foofs:1 {0=foofs.smithi027.xaykap=up:active} 1 up:standby 2024-06-01T02:48:45.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:45 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:48:45.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:48:45.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: pgmap v223: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 11 op/s 2024-06-01T02:48:45.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:45.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:45.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:48:45.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-06-01T02:48:45.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: Cluster is now healthy 2024-06-01T02:48:45.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670] up:boot 2024-06-01T02:48:45.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: fsmap foofs:1 {0=foofs.smithi027.xaykap=up:active} 1 up:standby 2024-06-01T02:48:45.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:45 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:48:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:48:47.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:46 smithi003 ceph-mon[67363]: pgmap v224: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.4 KiB/s wr, 10 op/s 2024-06-01T02:48:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:48:47.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:46 smithi027 ceph-mon[53801]: pgmap v224: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.4 KiB/s wr, 10 op/s 2024-06-01T02:48:49.086 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:48 smithi003 ceph-mon[67363]: pgmap v225: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 7.4 KiB/s wr, 12 op/s 2024-06-01T02:48:49.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:48 smithi027 ceph-mon[53801]: pgmap v225: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 7.4 KiB/s wr, 12 op/s 2024-06-01T02:48:51.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:50 smithi003 ceph-mon[67363]: pgmap v226: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 8.2 KiB/s wr, 11 op/s 2024-06-01T02:48:51.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:50 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:51.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:50 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:51.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:50 smithi027 ceph-mon[53801]: pgmap v226: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 8.2 KiB/s wr, 11 op/s 2024-06-01T02:48:51.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:50 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:51.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:50 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:52.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:52.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:52 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:52.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:52 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:52.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:52 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:53.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:53 smithi003 ceph-mon[67363]: pgmap v227: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 8.1 KiB/s wr, 4 op/s 2024-06-01T02:48:53.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:53 smithi027 ceph-mon[53801]: pgmap v227: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 8.1 KiB/s wr, 4 op/s 2024-06-01T02:48:55.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: pgmap v228: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 10 KiB/s wr, 5 op/s 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: Detected new or changed devices on smithi003 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:55.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:55.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi027.xaykap"]}]: dispatch 2024-06-01T02:48:55.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: pgmap v228: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 10 KiB/s wr, 5 op/s 2024-06-01T02:48:55.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: Detected new or changed devices on smithi003 2024-06-01T02:48:55.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:55.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:55.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:55.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:48:55.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:55.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:55.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:48:55.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:55.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:55.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:55.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:48:55.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi027.xaykap"]}]: dispatch 2024-06-01T02:48:56.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:56 smithi027 ceph-mon[53801]: Upgrade: It appears safe to stop mds.foofs.smithi027.xaykap 2024-06-01T02:48:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi027.xaykap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:48:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:56.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:56 smithi003 ceph-mon[67363]: Upgrade: It appears safe to stop mds.foofs.smithi027.xaykap 2024-06-01T02:48:56.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:48:56.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi027.xaykap", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-01T02:48:56.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:48:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:57 smithi027 ceph-mon[53801]: pgmap v229: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 10 KiB/s wr, 5 op/s 2024-06-01T02:48:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:57 smithi027 ceph-mon[53801]: Upgrade: Updating mds.foofs.smithi027.xaykap 2024-06-01T02:48:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:57 smithi027 ceph-mon[53801]: Deploying daemon mds.foofs.smithi027.xaykap on smithi027 2024-06-01T02:48:57.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:57 smithi003 ceph-mon[67363]: pgmap v229: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 10 KiB/s wr, 5 op/s 2024-06-01T02:48:57.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:57 smithi003 ceph-mon[67363]: Upgrade: Updating mds.foofs.smithi027.xaykap 2024-06-01T02:48:57.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:57 smithi003 ceph-mon[67363]: Deploying daemon mds.foofs.smithi027.xaykap on smithi027 2024-06-01T02:48:58.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003[67359]: 2024-06-01T02:48:58.547+0000 7fa4152df700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-01T02:48:58.837 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:48:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:48:58.570+0000 7f63a4cfc700 -1 bdev(0x55a6587b6000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 4 2024-06-01T02:48:58.837 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:48:58 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:48:58.571+0000 7f63a7501700 -1 bdev(0x55a6587b6000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 6 2024-06-01T02:48:58.852 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:48:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:48:58.568+0000 7ff5b1e2a700 -1 bdev(0x557f2ffa2000 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 4 2024-06-01T02:48:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:59 smithi003 ceph-mon[67363]: pgmap v230: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 9.1 KiB/s wr, 4 op/s 2024-06-01T02:48:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:59 smithi003 ceph-mon[67363]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-06-01T02:48:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:59 smithi003 ceph-mon[67363]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-01T02:48:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:59 smithi003 ceph-mon[67363]: osdmap e128: 8 total, 8 up, 8 in 2024-06-01T02:48:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:59 smithi003 ceph-mon[67363]: Standby daemon mds.foofs.smithi003.kexxec assigned to filesystem foofs as rank 0 2024-06-01T02:48:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:59 smithi003 ceph-mon[67363]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-06-01T02:48:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:59 smithi003 ceph-mon[67363]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-01T02:48:59.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:59 smithi003 ceph-mon[67363]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-06-01T02:48:59.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:48:59 smithi003 ceph-mon[67363]: fsmap foofs:1/1 {0=foofs.smithi003.kexxec=up:replay} 2024-06-01T02:48:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:59 smithi027 ceph-mon[53801]: pgmap v230: 97 pgs: 97 active+clean; 240 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 9.1 KiB/s wr, 4 op/s 2024-06-01T02:48:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:59 smithi027 ceph-mon[53801]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-06-01T02:48:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:59 smithi027 ceph-mon[53801]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-01T02:48:59.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:59 smithi027 ceph-mon[53801]: osdmap e128: 8 total, 8 up, 8 in 2024-06-01T02:48:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:59 smithi027 ceph-mon[53801]: Standby daemon mds.foofs.smithi003.kexxec assigned to filesystem foofs as rank 0 2024-06-01T02:48:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:59 smithi027 ceph-mon[53801]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-06-01T02:48:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:59 smithi027 ceph-mon[53801]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-01T02:48:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:59 smithi027 ceph-mon[53801]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-06-01T02:48:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:48:59 smithi027 ceph-mon[53801]: fsmap foofs:1/1 {0=foofs.smithi003.kexxec=up:replay} 2024-06-01T02:49:00.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:00 smithi003 ceph-mon[67363]: reconnect by client.14562 172.21.15.3:0/3874295321 after 0.000999987 2024-06-01T02:49:00.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:00 smithi003 ceph-mon[67363]: reconnect by client.24357 172.21.15.27:0/1146346812 after 0.000999987 2024-06-01T02:49:00.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:00 smithi003 ceph-mon[67363]: mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670] up:reconnect 2024-06-01T02:49:00.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:00 smithi003 ceph-mon[67363]: fsmap foofs:1/1 {0=foofs.smithi003.kexxec=up:reconnect} 2024-06-01T02:49:00.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:00 smithi027 ceph-mon[53801]: reconnect by client.14562 172.21.15.3:0/3874295321 after 0.000999987 2024-06-01T02:49:00.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:00 smithi027 ceph-mon[53801]: reconnect by client.24357 172.21.15.27:0/1146346812 after 0.000999987 2024-06-01T02:49:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:00 smithi027 ceph-mon[53801]: mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670] up:reconnect 2024-06-01T02:49:00.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:00 smithi027 ceph-mon[53801]: fsmap foofs:1/1 {0=foofs.smithi003.kexxec=up:reconnect} 2024-06-01T02:49:01.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:01 smithi003 ceph-mon[67363]: pgmap v232: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.7 KiB/s wr, 5 op/s 2024-06-01T02:49:01.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:01 smithi003 ceph-mon[67363]: mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670] up:rejoin 2024-06-01T02:49:01.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:01 smithi003 ceph-mon[67363]: fsmap foofs:1/1 {0=foofs.smithi003.kexxec=up:rejoin} 2024-06-01T02:49:01.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:01 smithi003 ceph-mon[67363]: daemon mds.foofs.smithi003.kexxec is now active in filesystem foofs as rank 0 2024-06-01T02:49:01.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:01.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:49:01.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:01 smithi027 ceph-mon[53801]: pgmap v232: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.7 KiB/s wr, 5 op/s 2024-06-01T02:49:01.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:01 smithi027 ceph-mon[53801]: mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670] up:rejoin 2024-06-01T02:49:01.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:01 smithi027 ceph-mon[53801]: fsmap foofs:1/1 {0=foofs.smithi003.kexxec=up:rejoin} 2024-06-01T02:49:01.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:01 smithi027 ceph-mon[53801]: daemon mds.foofs.smithi003.kexxec is now active in filesystem foofs as rank 0 2024-06-01T02:49:01.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:01.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:49:02.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:02 smithi003 ceph-mon[67363]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-06-01T02:49:02.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:02 smithi003 ceph-mon[67363]: mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670] up:active 2024-06-01T02:49:02.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:02 smithi003 ceph-mon[67363]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 2024-06-01T02:49:02.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:02 smithi027 ceph-mon[53801]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-06-01T02:49:02.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:02 smithi027 ceph-mon[53801]: mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670] up:active 2024-06-01T02:49:02.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:02 smithi027 ceph-mon[53801]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 2024-06-01T02:49:03.705 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:03 smithi027 ceph-mon[53801]: pgmap v233: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.7 KiB/s wr, 5 op/s 2024-06-01T02:49:03.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:03 smithi003 ceph-mon[67363]: pgmap v233: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.7 KiB/s wr, 5 op/s 2024-06-01T02:49:05.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:05 smithi003 ceph-mon[67363]: pgmap v234: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 511 B/s wr, 9 op/s 2024-06-01T02:49:05.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:05 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:05.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:05 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:05.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:05 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:05.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:05 smithi027 ceph-mon[53801]: pgmap v234: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 511 B/s wr, 9 op/s 2024-06-01T02:49:05.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:05 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:05.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:05 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:05.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:05 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:06.752 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:06 smithi027 ceph-mon[53801]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-06-01T02:49:06.752 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:06 smithi027 ceph-mon[53801]: Cluster is now healthy 2024-06-01T02:49:06.752 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:06 smithi027 ceph-mon[53801]: mds.? [v2:172.21.15.27:6832/2801932043,v1:172.21.15.27:6833/2801932043] up:boot 2024-06-01T02:49:06.752 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:06 smithi027 ceph-mon[53801]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:49:06.752 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:06 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:49:06.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:06 smithi003 ceph-mon[67363]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-06-01T02:49:06.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:06 smithi003 ceph-mon[67363]: Cluster is now healthy 2024-06-01T02:49:06.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:06 smithi003 ceph-mon[67363]: mds.? [v2:172.21.15.27:6832/2801932043,v1:172.21.15.27:6833/2801932043] up:boot 2024-06-01T02:49:06.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:06 smithi003 ceph-mon[67363]: fsmap foofs:1 {0=foofs.smithi003.kexxec=up:active} 1 up:standby 2024-06-01T02:49:06.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:06 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:49:07.789 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:07 smithi027 ceph-mon[53801]: pgmap v235: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 511 B/s wr, 11 op/s 2024-06-01T02:49:07.836 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:07 smithi003 ceph-mon[67363]: pgmap v235: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 511 B/s wr, 11 op/s 2024-06-01T02:49:09.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:09 smithi003 ceph-mon[67363]: pgmap v236: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 8.9 KiB/s wr, 15 op/s 2024-06-01T02:49:09.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:09 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:09.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:09 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:09 smithi027 ceph-mon[53801]: pgmap v236: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 8.9 KiB/s wr, 15 op/s 2024-06-01T02:49:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:09 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:09 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:10.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:10 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:10.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:10 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:10.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:10 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:10.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:10 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:11.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:11 smithi027 ceph-mon[53801]: pgmap v237: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 7.8 KiB/s wr, 13 op/s 2024-06-01T02:49:11.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:11 smithi003 ceph-mon[67363]: pgmap v237: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 7.8 KiB/s wr, 13 op/s 2024-06-01T02:49:13.706 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: pgmap v238: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 7.3 KiB/s wr, 9 op/s 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.707 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi003.kexxec"}]': finished 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi027.xaykap"}]': finished 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.708 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:13 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: pgmap v238: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.5 MiB/s rd, 7.3 KiB/s wr, 9 op/s 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi003.kexxec"}]: dispatch 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi003.kexxec"}]': finished 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi027.xaykap"}]: dispatch 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi027.xaykap"}]': finished 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:13.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:13.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:13 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:14.465 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:49:14.711 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:14 smithi003 ceph-mon[67363]: Detected new or changed devices on smithi027 2024-06-01T02:49:14.711 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:14 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all mds 2024-06-01T02:49:14.711 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:14 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all rgw 2024-06-01T02:49:14.711 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:14 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all rbd-mirror 2024-06-01T02:49:14.711 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:14 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all ceph-exporter 2024-06-01T02:49:14.711 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:14 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all iscsi 2024-06-01T02:49:14.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:14 smithi027 ceph-mon[53801]: Detected new or changed devices on smithi027 2024-06-01T02:49:14.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:14 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all mds 2024-06-01T02:49:14.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:14 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all rgw 2024-06-01T02:49:14.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:14 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all rbd-mirror 2024-06-01T02:49:14.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:14 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all ceph-exporter 2024-06-01T02:49:14.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:14 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all iscsi 2024-06-01T02:49:15.179 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (7m) 25s ago 31m 14.0M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (5m) 25s ago 31m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (5m) 6s ago 30m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (7m) 25s ago 31m 44.3M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (31s) 25s ago 25m 13.7M - 19.0.0-2946-g3d89accf b045ac545f82 00d739c1b4e3 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (10s) 6s ago 25m 12.1M - 19.0.0-2946-g3d89accf b045ac545f82 459ab19de6be 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (7m) 25s ago 32m 513M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (6m) 6s ago 30m 454M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (6m) 25s ago 32m 58.1M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (6m) 6s ago 30m 44.6M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (25m) 25s ago 25m 290M - 4.0 e1d6a67b021e 3b935ad683f4 2024-06-01T02:49:15.180 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (25m) 6s ago 25m 90.7M - 4.0 e1d6a67b021e e67c0ae31d22 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (7m) 25s ago 30m 9407k - 1.3.1 1dbe0e931976 4e7a55148511 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (7m) 6s ago 30m 18.1M - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (5m) 25s ago 29m 200M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (4m) 25s ago 29m 184M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (3m) 25s ago 28m 156M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (3m) 25s ago 28m 194M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (2m) 6s ago 28m 204M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (2m) 6s ago 27m 196M 2704M 19.0.0-2946-g3d89accf b045ac545f82 cce965464b24 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (89s) 6s ago 27m 188M 2704M 19.0.0-2946-g3d89accf b045ac545f82 13f0723f71f7 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (54s) 6s ago 26m 230M 2704M 19.0.0-2946-g3d89accf b045ac545f82 196fb45013fd 2024-06-01T02:49:15.181 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (6m) 25s ago 30m 70.0M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:49:15.506 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: pgmap v239: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 7.3 KiB/s wr, 9 op/s 2024-06-01T02:49:15.506 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:15.506 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:15.506 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-01T02:49:15.507 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-01T02:49:15.507 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-01T02:49:15.507 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:15.507 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-01T02:49:15.507 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-01T02:49:15.507 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-01T02:49:15.507 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:15.605 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:49:15.605 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:49:15.605 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:49:15.605 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:49:15.605 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:49:15.605 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:49:15.605 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 8 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 14 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:49:15.606 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: pgmap v239: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 7.3 KiB/s wr, 9 op/s 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-01T02:49:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi003.dmybfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-01T02:49:15.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:15.963 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:49:15.963 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:49:15.963 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:49:15.963 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:49:15.963 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:49:15.963 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:49:15.964 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:49:15.964 INFO:teuthology.orchestra.run.smithi003.stdout: "mds", 2024-06-01T02:49:15.964 INFO:teuthology.orchestra.run.smithi003.stdout: "osd", 2024-06-01T02:49:15.964 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr" 2024-06-01T02:49:15.964 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:49:15.964 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "16/23 daemons upgraded", 2024-06-01T02:49:15.964 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading nfs daemons", 2024-06-01T02:49:15.964 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:49:15.964 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:49:16.379 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:49:16.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: Upgrade: Updating nfs.foo.0.0.smithi003.dmybfb (1/2) 2024-06-01T02:49:16.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: Creating key for client.nfs.foo.0.0.smithi003.dmybfb 2024-06-01T02:49:16.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-06-01T02:49:16.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:16.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: Rados config object exists: conf-nfs.foo 2024-06-01T02:49:16.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: Creating key for client.nfs.foo.0.0.smithi003.dmybfb-rgw 2024-06-01T02:49:16.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: Bind address in nfs.foo.0.0.smithi003.dmybfb's ganesha conf is defaulting to empty 2024-06-01T02:49:16.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: Deploying daemon nfs.foo.0.0.smithi003.dmybfb on smithi003 2024-06-01T02:49:16.503 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:16.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/2871233221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:16.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:16.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:49:16.504 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:16 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1800134194' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: Upgrade: Updating nfs.foo.0.0.smithi003.dmybfb (1/2) 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: Creating key for client.nfs.foo.0.0.smithi003.dmybfb 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: Rados config object exists: conf-nfs.foo 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: Creating key for client.nfs.foo.0.0.smithi003.dmybfb-rgw 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: Bind address in nfs.foo.0.0.smithi003.dmybfb's ganesha conf is defaulting to empty 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: Deploying daemon nfs.foo.0.0.smithi003.dmybfb on smithi003 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/2871233221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:16.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:49:16.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:16 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/1800134194' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:49:17.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:17 smithi003 ceph-mon[67363]: pgmap v240: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 684 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-06-01T02:49:17.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:17 smithi003 ceph-mon[67363]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:17.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:17 smithi027 ceph-mon[53801]: pgmap v240: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 684 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-06-01T02:49:17.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:17 smithi027 ceph-mon[53801]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:19.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:19 smithi027 ceph-mon[53801]: pgmap v241: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 7.5 KiB/s wr, 4 op/s 2024-06-01T02:49:19.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:19 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:19.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:19 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:19.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:19 smithi003 ceph-mon[67363]: pgmap v241: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 7.5 KiB/s wr, 4 op/s 2024-06-01T02:49:19.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:19 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:19.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:19 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:21.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: pgmap v242: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 0 op/s 2024-06-01T02:49:21.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: Upgrade: Updating nfs.foo.1.0.smithi027.dsjwox (2/2) 2024-06-01T02:49:21.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:21.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: Creating key for client.nfs.foo.1.0.smithi027.dsjwox 2024-06-01T02:49:21.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-01T02:49:21.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-06-01T02:49:21.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: Rados config object exists: conf-nfs.foo 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: Creating key for client.nfs.foo.1.0.smithi027.dsjwox-rgw 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: Bind address in nfs.foo.1.0.smithi027.dsjwox's ganesha conf is defaulting to empty 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:21.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:21 smithi003 ceph-mon[67363]: Deploying daemon nfs.foo.1.0.smithi027.dsjwox on smithi027 2024-06-01T02:49:21.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: pgmap v242: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 0 op/s 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: Upgrade: Updating nfs.foo.1.0.smithi027.dsjwox (2/2) 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: Creating key for client.nfs.foo.1.0.smithi027.dsjwox 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: Rados config object exists: conf-nfs.foo 2024-06-01T02:49:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: Creating key for client.nfs.foo.1.0.smithi027.dsjwox-rgw 2024-06-01T02:49:21.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi027.dsjwox-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-01T02:49:21.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: Bind address in nfs.foo.1.0.smithi027.dsjwox's ganesha conf is defaulting to empty 2024-06-01T02:49:21.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:21.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:21 smithi027 ceph-mon[53801]: Deploying daemon nfs.foo.1.0.smithi027.dsjwox on smithi027 2024-06-01T02:49:23.551 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:23 smithi027 ceph-mon[53801]: pgmap v243: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 0 op/s 2024-06-01T02:49:23.551 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:23 smithi027 ceph-mon[53801]: from='mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670]' entity='mds.foofs.smithi003.kexxec' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.3:0/3874295321"}]: dispatch 2024-06-01T02:49:23.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:23 smithi003 ceph-mon[67363]: pgmap v243: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 0 op/s 2024-06-01T02:49:23.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:23 smithi003 ceph-mon[67363]: from='mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670]' entity='mds.foofs.smithi003.kexxec' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.3:0/3874295321"}]: dispatch 2024-06-01T02:49:24.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:24 smithi003 ceph-mon[67363]: Evicting (and blocklisting) client session 14562 (172.21.15.3:0/3874295321) 2024-06-01T02:49:24.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:24 smithi003 ceph-mon[67363]: from='mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670]' entity='mds.foofs.smithi003.kexxec' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.3:0/3874295321"}]': finished 2024-06-01T02:49:24.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:24 smithi003 ceph-mon[67363]: osdmap e129: 8 total, 8 up, 8 in 2024-06-01T02:49:24.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:24 smithi003 ceph-mon[67363]: from='mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670]' entity='mds.foofs.smithi003.kexxec' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.27:0/1146346812"}]: dispatch 2024-06-01T02:49:24.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:24 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:24.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:24 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:24.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:24 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:24.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:24 smithi027 ceph-mon[53801]: Evicting (and blocklisting) client session 14562 (172.21.15.3:0/3874295321) 2024-06-01T02:49:24.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:24 smithi027 ceph-mon[53801]: from='mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670]' entity='mds.foofs.smithi003.kexxec' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.3:0/3874295321"}]': finished 2024-06-01T02:49:24.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:24 smithi027 ceph-mon[53801]: osdmap e129: 8 total, 8 up, 8 in 2024-06-01T02:49:24.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:24 smithi027 ceph-mon[53801]: from='mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670]' entity='mds.foofs.smithi003.kexxec' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.27:0/1146346812"}]: dispatch 2024-06-01T02:49:24.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:24 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:24.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:24 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:24.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:24 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:25.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:25 smithi003 ceph-mon[67363]: pgmap v245: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 716 B/s wr, 2 op/s 2024-06-01T02:49:25.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:25 smithi003 ceph-mon[67363]: Evicting (and blocklisting) client session 24357 (172.21.15.27:0/1146346812) 2024-06-01T02:49:25.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:25 smithi003 ceph-mon[67363]: from='mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670]' entity='mds.foofs.smithi003.kexxec' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.27:0/1146346812"}]': finished 2024-06-01T02:49:25.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:25 smithi003 ceph-mon[67363]: osdmap e130: 8 total, 8 up, 8 in 2024-06-01T02:49:25.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:25 smithi027 ceph-mon[53801]: pgmap v245: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 716 B/s wr, 2 op/s 2024-06-01T02:49:25.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:25 smithi027 ceph-mon[53801]: Evicting (and blocklisting) client session 24357 (172.21.15.27:0/1146346812) 2024-06-01T02:49:25.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:25 smithi027 ceph-mon[53801]: from='mds.? [v2:172.21.15.3:6834/1878822670,v1:172.21.15.3:6835/1878822670]' entity='mds.foofs.smithi003.kexxec' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.27:0/1146346812"}]': finished 2024-06-01T02:49:25.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:25 smithi027 ceph-mon[53801]: osdmap e130: 8 total, 8 up, 8 in 2024-06-01T02:49:27.692 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:27 smithi003 ceph-mon[67363]: pgmap v247: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.4 KiB/s rd, 511 B/s wr, 2 op/s 2024-06-01T02:49:27.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:27 smithi027 ceph-mon[53801]: pgmap v247: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.4 KiB/s rd, 511 B/s wr, 2 op/s 2024-06-01T02:49:28.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:28 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:28.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:28 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:28.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:28 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:28.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:28 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:29.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:29 smithi027 ceph-mon[53801]: pgmap v248: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-06-01T02:49:29.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:29.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:29 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:29.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:29 smithi003 ceph-mon[67363]: pgmap v248: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-06-01T02:49:29.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:29.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:29 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:30.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:30.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:30 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:30.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:30 smithi027 ceph-mon[53801]: pgmap v249: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-06-01T02:49:31.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:31.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:30 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:31.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:30 smithi003 ceph-mon[67363]: pgmap v249: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-06-01T02:49:32.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:32.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:32.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:49:32.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:32.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:32.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:49:33.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:33 smithi003 ceph-mon[67363]: pgmap v250: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.8 KiB/s wr, 4 op/s 2024-06-01T02:49:33.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:33 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:33.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:33 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:33.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:33 smithi027 ceph-mon[53801]: pgmap v250: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.8 KiB/s wr, 4 op/s 2024-06-01T02:49:33.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:33 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:33.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:33 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.463 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: pgmap v251: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-06-01T02:49:35.463 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.463 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.463 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:35.463 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:49:35.463 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.463 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.464 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all nfs 2024-06-01T02:49:35.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi003.dmybfb"}]: dispatch 2024-06-01T02:49:35.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi003.dmybfb"}]': finished 2024-06-01T02:49:35.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi027.dsjwox"}]: dispatch 2024-06-01T02:49:35.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi027.dsjwox"}]': finished 2024-06-01T02:49:35.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: Upgrade: Setting container_image for all nvmeof 2024-06-01T02:49:35.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:35 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: pgmap v251: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-06-01T02:49:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:49:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all nfs 2024-06-01T02:49:35.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:35.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi003.dmybfb"}]: dispatch 2024-06-01T02:49:35.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi003.dmybfb"}]': finished 2024-06-01T02:49:35.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi027.dsjwox"}]: dispatch 2024-06-01T02:49:35.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi027.dsjwox"}]': finished 2024-06-01T02:49:35.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:35.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: Upgrade: Setting container_image for all nvmeof 2024-06-01T02:49:35.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:35 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:37 smithi003 ceph-mon[67363]: pgmap v252: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-06-01T02:49:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:37 smithi003 ceph-mon[67363]: Upgrade: Updating node-exporter.smithi003 (1/2) 2024-06-01T02:49:37.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:37 smithi003 ceph-mon[67363]: Deploying daemon node-exporter.smithi003 on smithi003 2024-06-01T02:49:37.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:37 smithi027 ceph-mon[53801]: pgmap v252: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-06-01T02:49:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:37 smithi027 ceph-mon[53801]: Upgrade: Updating node-exporter.smithi003 (1/2) 2024-06-01T02:49:37.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:37 smithi027 ceph-mon[53801]: Deploying daemon node-exporter.smithi003 on smithi003 2024-06-01T02:49:39.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:39 smithi003 ceph-mon[67363]: pgmap v253: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-06-01T02:49:39.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:39 smithi027 ceph-mon[53801]: pgmap v253: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-06-01T02:49:41.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:41 smithi003 ceph-mon[67363]: pgmap v254: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 426 B/s wr, 1 op/s 2024-06-01T02:49:41.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:41 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:41.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:41 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:41.842 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:41 smithi027 ceph-mon[53801]: pgmap v254: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 426 B/s wr, 1 op/s 2024-06-01T02:49:41.842 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:41 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:41.842 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:41 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:43.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:43 smithi003 ceph-mon[67363]: pgmap v255: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 426 B/s wr, 1 op/s 2024-06-01T02:49:43.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:43 smithi027 ceph-mon[53801]: pgmap v255: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 426 B/s wr, 1 op/s 2024-06-01T02:49:44.832 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:44 smithi027 ceph-mon[53801]: Upgrade: Updating node-exporter.smithi027 (2/2) 2024-06-01T02:49:44.832 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:44 smithi027 ceph-mon[53801]: Deploying daemon node-exporter.smithi027 on smithi027 2024-06-01T02:49:44.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:44 smithi003 ceph-mon[67363]: Upgrade: Updating node-exporter.smithi027 (2/2) 2024-06-01T02:49:44.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:44 smithi003 ceph-mon[67363]: Deploying daemon node-exporter.smithi027 on smithi027 2024-06-01T02:49:45.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:45 smithi003 ceph-mon[67363]: pgmap v256: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-06-01T02:49:45.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:45 smithi027 ceph-mon[53801]: pgmap v256: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-06-01T02:49:46.758 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:49:46.758 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:49:46.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:49:47.443 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:49:47.443 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (8m) 17s ago 31m 14.0M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:49:47.443 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (6m) 17s ago 31m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:49:47.443 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (6m) 19s ago 31m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:49:47.443 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (8m) 17s ago 31m 44.7M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:49:47.443 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (63s) 17s ago 26m 30.9M - 19.0.0-2946-g3d89accf b045ac545f82 00d739c1b4e3 2024-06-01T02:49:47.443 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (42s) 19s ago 25m 12.2M - 19.0.0-2946-g3d89accf b045ac545f82 459ab19de6be 2024-06-01T02:49:47.443 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (7m) 17s ago 33m 514M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:49:47.443 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (7m) 19s ago 31m 454M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (7m) 17s ago 33m 59.2M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (6m) 19s ago 31m 45.8M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (29s) 17s ago 25m 63.4M - 5.5 b045ac545f82 f0c3e06ef6f3 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (23s) 19s ago 25m 60.8M - 5.5 b045ac545f82 92e8b05a633d 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 starting - - - - 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (8m) 19s ago 30m 21.4M - 1.3.1 1dbe0e931976 199feea2baaa 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (5m) 17s ago 30m 201M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (5m) 17s ago 29m 187M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (4m) 17s ago 29m 157M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:49:47.444 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (3m) 17s ago 29m 195M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:49:47.445 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (3m) 19s ago 28m 204M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:49:47.445 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (2m) 19s ago 28m 196M 2704M 19.0.0-2946-g3d89accf b045ac545f82 cce965464b24 2024-06-01T02:49:47.445 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (2m) 19s ago 27m 189M 2704M 19.0.0-2946-g3d89accf b045ac545f82 13f0723f71f7 2024-06-01T02:49:47.445 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (86s) 19s ago 27m 230M 2704M 19.0.0-2946-g3d89accf b045ac545f82 196fb45013fd 2024-06-01T02:49:47.445 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (7m) 17s ago 30m 70.2M - 2.33.4 514e6a882f6e c3f6e9a522b9 2024-06-01T02:49:47.617 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:47 smithi027 ceph-mon[53801]: pgmap v257: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-06-01T02:49:47.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:47 smithi003 ceph-mon[67363]: pgmap v257: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-06-01T02:49:47.861 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:49:47.861 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 8 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:49:47.862 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:49:47.863 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 14 2024-06-01T02:49:47.863 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:49:47.863 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:49:48.219 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:49:48.219 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "mds", 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "osd", 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr", 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "nfs" 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "18/23 daemons upgraded", 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading node-exporter daemons", 2024-06-01T02:49:48.220 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:49:48.221 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:49:48.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:48 smithi003 ceph-mon[67363]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:48.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:48 smithi003 ceph-mon[67363]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:48.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:48 smithi003 ceph-mon[67363]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:48.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:48 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/293731200' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:48.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:48 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:48.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:48 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:48.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:48 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:48.655 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:49:48.758 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:48 smithi027 ceph-mon[53801]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:48.758 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:48 smithi027 ceph-mon[53801]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:48.758 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:48 smithi027 ceph-mon[53801]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:48.758 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:48 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/293731200' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:48.759 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:48 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:48.759 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:48 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:48.759 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:48 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:49.682 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:49 smithi027 ceph-mon[53801]: pgmap v258: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 341 B/s wr, 1 op/s 2024-06-01T02:49:49.683 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:49 smithi027 ceph-mon[53801]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:49.683 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:49 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3919419200' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:49:49.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:49 smithi003 ceph-mon[67363]: pgmap v258: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 853 B/s rd, 341 B/s wr, 1 op/s 2024-06-01T02:49:49.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:49 smithi003 ceph-mon[67363]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:49:49.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:49 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3919419200' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:49:51.763 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:51 smithi003 ceph-mon[67363]: pgmap v259: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-06-01T02:49:51.764 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:51 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:51.764 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:51 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:51.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:51 smithi027 ceph-mon[53801]: pgmap v259: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-06-01T02:49:51.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:51 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:51.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:51 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:53.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:53 smithi027 ceph-mon[53801]: pgmap v260: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:49:53.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:53 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:53.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:53 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:53.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:53 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:53.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:53 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:53.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:53 smithi003 ceph-mon[67363]: pgmap v260: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:49:53.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:53 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:53.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:53 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:53.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:53 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:53.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:53 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:55.339 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:55 smithi027 ceph-mon[53801]: pgmap v261: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:49:55.339 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:55.339 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:55 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:55 smithi003 ceph-mon[67363]: pgmap v261: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:49:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:55 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:56.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:56.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:56 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:56.838 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:56 smithi003 ceph-mon[67363]: pgmap v262: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:49:57.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:57.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:56 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:57.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:56 smithi027 ceph-mon[53801]: pgmap v262: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:49:58.921 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:58.921 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:58.921 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:58.921 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:49:58.921 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:58.921 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:58.921 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: pgmap v263: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:49:58.921 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:58.921 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:58.922 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:58 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:59.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:59.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:49:59.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:49:59.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:59.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:49:59.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: pgmap v263: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:49:59.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:49:59.104 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:58 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:00.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:49:59 smithi027 ceph-mon[53801]: Upgrade: Updating prometheus.smithi003 2024-06-01T02:50:00.285 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:49:59 smithi003 ceph-mon[67363]: Upgrade: Updating prometheus.smithi003 2024-06-01T02:50:01.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:00 smithi027 ceph-mon[53801]: Deploying daemon prometheus.smithi003 on smithi003 2024-06-01T02:50:01.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:00 smithi027 ceph-mon[53801]: pgmap v264: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:01.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:00 smithi027 ceph-mon[53801]: overall HEALTH_OK 2024-06-01T02:50:01.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:50:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:00 smithi003 ceph-mon[67363]: Deploying daemon prometheus.smithi003 on smithi003 2024-06-01T02:50:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:00 smithi003 ceph-mon[67363]: pgmap v264: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:00 smithi003 ceph-mon[67363]: overall HEALTH_OK 2024-06-01T02:50:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:50:03.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:03 smithi003 ceph-mon[67363]: pgmap v265: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:03.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:03 smithi027 ceph-mon[53801]: pgmap v265: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:05.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:05 smithi003 ceph-mon[67363]: pgmap v266: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:05.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:05 smithi027 ceph-mon[53801]: pgmap v266: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:07 smithi003 ceph-mon[67363]: pgmap v267: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:07.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:07 smithi027 ceph-mon[53801]: pgmap v267: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:09.484 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:09 smithi003 ceph-mon[67363]: pgmap v268: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:09.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:09 smithi027 ceph-mon[53801]: pgmap v268: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:11.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:11 smithi003 ceph-mon[67363]: pgmap v269: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:11.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:11 smithi027 ceph-mon[53801]: pgmap v269: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:13.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:13 smithi003 ceph-mon[67363]: pgmap v270: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:13.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:13 smithi027 ceph-mon[53801]: pgmap v270: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:15.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:15 smithi003 ceph-mon[67363]: pgmap v271: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:15.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:15.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:15.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:15 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:50:15.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:15 smithi027 ceph-mon[53801]: pgmap v271: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:15.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:15.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:15 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:50:16.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:50:16.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:50:17.819 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:17 smithi003 ceph-mon[67363]: pgmap v272: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:17.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:17 smithi027 ceph-mon[53801]: pgmap v272: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:19.066 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:50:19.771 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (8m) 0s ago 32m 20.5M - 0.23.0 ba2b418f427c 106eb0919f19 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (6m) 0s ago 32m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (6m) 28s ago 31m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (8m) 0s ago 32m 45.3M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (95s) 0s ago 26m 31.2M - 19.0.0-2946-g3d89accf b045ac545f82 00d739c1b4e3 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (75s) 28s ago 26m 12.3M - 19.0.0-2946-g3d89accf b045ac545f82 459ab19de6be 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (8m) 0s ago 33m 517M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (7m) 28s ago 31m 454M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (7m) 0s ago 33m 60.9M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (7m) 28s ago 31m 45.6M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (61s) 0s ago 26m 64.2M - 5.5 b045ac545f82 f0c3e06ef6f3 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (55s) 28s ago 26m 61.2M - 5.5 b045ac545f82 92e8b05a633d 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (38s) 0s ago 32m 8102k - 1.5.0 0da6a335fe13 087d7399c180 2024-06-01T02:50:19.772 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (32s) 28s ago 31m 3497k - 1.5.0 0da6a335fe13 6b2d846d4d14 2024-06-01T02:50:19.773 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (6m) 0s ago 31m 202M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:50:19.773 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (5m) 0s ago 30m 187M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:50:19.773 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (4m) 0s ago 30m 157M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:50:19.773 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (4m) 0s ago 29m 196M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:50:19.773 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (3m) 28s ago 29m 204M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:50:19.773 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (3m) 28s ago 28m 196M 2704M 19.0.0-2946-g3d89accf b045ac545f82 cce965464b24 2024-06-01T02:50:19.773 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (2m) 28s ago 28m 189M 2704M 19.0.0-2946-g3d89accf b045ac545f82 13f0723f71f7 2024-06-01T02:50:19.773 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (118s) 28s ago 28m 230M 2704M 19.0.0-2946-g3d89accf b045ac545f82 196fb45013fd 2024-06-01T02:50:19.773 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (6s) 0s ago 31m 45.6M - 2.43.0 a07b618ecd1d fe24a7ed223d 2024-06-01T02:50:19.773 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:19 smithi003 ceph-mon[67363]: pgmap v273: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:19.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:19 smithi027 ceph-mon[53801]: pgmap v273: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:20.191 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:50:20.191 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:50:20.191 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:50:20.191 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:50:20.191 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:50:20.191 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:50:20.191 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:50:20.191 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:50:20.191 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 8 2024-06-01T02:50:20.192 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:50:20.192 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:50:20.192 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:50:20.192 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:50:20.192 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:50:20.192 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 14 2024-06-01T02:50:20.192 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:50:20.192 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout: "mds", 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout: "osd", 2024-06-01T02:50:20.556 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr", 2024-06-01T02:50:20.557 INFO:teuthology.orchestra.run.smithi003.stdout: "nfs" 2024-06-01T02:50:20.557 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:50:20.557 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "18/23 daemons upgraded", 2024-06-01T02:50:20.557 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading prometheus daemons", 2024-06-01T02:50:20.557 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:50:20.557 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:50:20.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:20 smithi003 ceph-mon[67363]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:20.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:20 smithi003 ceph-mon[67363]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:20.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:20.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:20 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:20.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:20 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1712288718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:20.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:20 smithi027 ceph-mon[53801]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:20.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:20 smithi027 ceph-mon[53801]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:20.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:20.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:20 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:20.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:20 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/1712288718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:20.978 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:50:21.773 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:21 smithi003 ceph-mon[67363]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:21.773 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:21 smithi003 ceph-mon[67363]: pgmap v274: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:21.773 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:21 smithi003 ceph-mon[67363]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:21.773 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:21.773 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:21 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:21.773 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:21 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3654469463' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:50:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:21 smithi027 ceph-mon[53801]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:21 smithi027 ceph-mon[53801]: pgmap v274: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:21 smithi027 ceph-mon[53801]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:21 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:21.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:21 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3654469463' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:50:23.654 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:23 smithi003 ceph-mon[67363]: pgmap v275: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:23.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:23 smithi027 ceph-mon[53801]: pgmap v275: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:25.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: pgmap v276: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:25.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:25.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:25.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:50:25.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:50:25.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:25.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:25.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:50:25.465 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.466 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.467 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:25 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: pgmap v276: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:25.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:25.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:25.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:50:25.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:50:25.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.854 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:25.855 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:25 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:27 smithi003 ceph-mon[67363]: pgmap v277: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:27 smithi003 ceph-mon[67363]: Upgrade: Updating alertmanager.smithi003 2024-06-01T02:50:27.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:27 smithi003 ceph-mon[67363]: Deploying daemon alertmanager.smithi003 on smithi003 2024-06-01T02:50:27.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:27 smithi027 ceph-mon[53801]: pgmap v277: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:27.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:27 smithi027 ceph-mon[53801]: Upgrade: Updating alertmanager.smithi003 2024-06-01T02:50:27.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:27 smithi027 ceph-mon[53801]: Deploying daemon alertmanager.smithi003 on smithi003 2024-06-01T02:50:29.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:29 smithi003 ceph-mon[67363]: pgmap v278: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:29.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:29 smithi027 ceph-mon[53801]: pgmap v278: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:31 smithi003 ceph-mon[67363]: pgmap v279: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:50:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:31 smithi027 ceph-mon[53801]: pgmap v279: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:31.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:50:33.529 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:33 smithi003 ceph-mon[67363]: pgmap v280: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:33.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:33 smithi027 ceph-mon[53801]: pgmap v280: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:35.659 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:35 smithi003 ceph-mon[67363]: pgmap v281: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:35.852 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:35 smithi027 ceph-mon[53801]: pgmap v281: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:37.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:37.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:37.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:36 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:50:37.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:36 smithi003 ceph-mon[67363]: pgmap v282: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:37.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:37.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:37.102 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:36 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:50:37.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:36 smithi027 ceph-mon[53801]: pgmap v282: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:39.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:38 smithi003 ceph-mon[67363]: pgmap v283: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:39.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:38 smithi027 ceph-mon[53801]: pgmap v283: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:41.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:40 smithi003 ceph-mon[67363]: pgmap v284: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:41.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:40 smithi027 ceph-mon[53801]: pgmap v284: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:42.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:42 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:42.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:42 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:42.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:42 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:42.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:42 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:43.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:43 smithi003 ceph-mon[67363]: pgmap v285: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:43.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:43 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:43.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:43 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:43 smithi027 ceph-mon[53801]: pgmap v285: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:43 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:43.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:43 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:45.400 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:45 smithi003 ceph-mon[67363]: pgmap v286: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:45.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:45 smithi027 ceph-mon[53801]: pgmap v286: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.589 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:50:46.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:46.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:46.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:50:46.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:50:46.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:46.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.604 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.605 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.605 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.605 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.605 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:46.605 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:47.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:47 smithi003 ceph-mon[67363]: pgmap v287: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:47.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:47 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:50:47.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:47 smithi027 ceph-mon[53801]: pgmap v287: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:47.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:47 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-01T02:50:49.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:49 smithi027 ceph-mon[53801]: pgmap v288: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:49.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:49 smithi027 ceph-mon[53801]: Upgrade: Updating grafana.smithi003 2024-06-01T02:50:49.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:49 smithi027 ceph-mon[53801]: Deploying daemon grafana.smithi003 on smithi003 2024-06-01T02:50:49.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:49 smithi003 ceph-mon[67363]: pgmap v288: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:49.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:49 smithi003 ceph-mon[67363]: Upgrade: Updating grafana.smithi003 2024-06-01T02:50:49.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:49 smithi003 ceph-mon[67363]: Deploying daemon grafana.smithi003 on smithi003 2024-06-01T02:50:51.412 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:50:51.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:51 smithi027 ceph-mon[53801]: pgmap v289: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:51.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:51 smithi003 ceph-mon[67363]: pgmap v289: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:52.112 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:50:52.112 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (16s) 10s ago 32m 17.0M - 0.25.0 c8568f914cd2 be8d55fb77d3 2024-06-01T02:50:52.112 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (7m) 10s ago 32m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (7m) 61s ago 32m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (9m) 10s ago 32m 45.9M - 8.3.5 dad864ee21e9 596bac145cea 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (2m) 10s ago 27m 31.4M - 19.0.0-2946-g3d89accf b045ac545f82 00d739c1b4e3 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (107s) 61s ago 27m 12.3M - 19.0.0-2946-g3d89accf b045ac545f82 459ab19de6be 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (9m) 10s ago 34m 517M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (8m) 61s ago 32m 454M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (8m) 10s ago 34m 61.1M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (7m) 61s ago 32m 45.6M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (93s) 10s ago 26m 64.4M - 5.5 b045ac545f82 f0c3e06ef6f3 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (88s) 61s ago 26m 61.2M - 5.5 b045ac545f82 92e8b05a633d 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (71s) 10s ago 32m 7860k - 1.5.0 0da6a335fe13 087d7399c180 2024-06-01T02:50:52.113 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (64s) 61s ago 32m 3497k - 1.5.0 0da6a335fe13 6b2d846d4d14 2024-06-01T02:50:52.114 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (6m) 10s ago 31m 202M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:50:52.114 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (6m) 10s ago 31m 187M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:50:52.114 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (5m) 10s ago 30m 157M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:50:52.114 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (4m) 10s ago 30m 196M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:50:52.114 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (4m) 61s ago 29m 204M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:50:52.114 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (3m) 61s ago 29m 196M 2704M 19.0.0-2946-g3d89accf b045ac545f82 cce965464b24 2024-06-01T02:50:52.114 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (3m) 61s ago 28m 189M 2704M 19.0.0-2946-g3d89accf b045ac545f82 13f0723f71f7 2024-06-01T02:50:52.114 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (2m) 61s ago 28m 230M 2704M 19.0.0-2946-g3d89accf b045ac545f82 196fb45013fd 2024-06-01T02:50:52.114 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (39s) 10s ago 31m 48.7M - 2.43.0 a07b618ecd1d fe24a7ed223d 2024-06-01T02:50:52.546 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:50:52.546 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:50:52.546 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:50:52.546 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 8 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 14 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:50:52.547 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:50:52.548 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:52 smithi003 ceph-mon[67363]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:52.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:52 smithi027 ceph-mon[53801]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:52.911 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": true, 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [ 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "mon", 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "crash", 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "mds", 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "osd", 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr", 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "nfs" 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": "18/23 daemons upgraded", 2024-06-01T02:50:52.912 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "Currently upgrading grafana daemons", 2024-06-01T02:50:52.913 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:50:52.913 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:50:53.365 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:50:53.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:53 smithi027 ceph-mon[53801]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:53.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:53 smithi027 ceph-mon[53801]: pgmap v290: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:53.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:53 smithi027 ceph-mon[53801]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:53.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:53 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/512766572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:53.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:53 smithi003 ceph-mon[67363]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:53.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:53 smithi003 ceph-mon[67363]: pgmap v290: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:53.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:53 smithi003 ceph-mon[67363]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:53.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:53 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/512766572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:50:54.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:54 smithi027 ceph-mon[53801]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:54.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:54 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3548719086' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:50:54.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:54 smithi003 ceph-mon[67363]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:50:54.837 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:54 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3548719086' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:50:55.606 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:55 smithi003 ceph-mon[67363]: pgmap v291: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:55.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:55 smithi027 ceph-mon[53801]: pgmap v291: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:57.705 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:57 smithi003 ceph-mon[67363]: pgmap v292: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:57.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:57 smithi027 ceph-mon[53801]: pgmap v292: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:50:59.625 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:50:59 smithi003 ceph-mon[67363]: pgmap v293: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:50:59.853 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:50:59 smithi027 ceph-mon[53801]: pgmap v293: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:00 smithi003 ceph-mon[67363]: pgmap v294: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:51:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:01.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:00 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:51:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:00 smithi027 ceph-mon[53801]: pgmap v294: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:51:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:01.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:00 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:51:03.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:02 smithi003 ceph-mon[67363]: pgmap v295: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:03.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:02 smithi027 ceph-mon[53801]: pgmap v295: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:05.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:04 smithi003 ceph-mon[67363]: pgmap v296: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:05.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:04 smithi027 ceph-mon[53801]: pgmap v296: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:07.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:06 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:07.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:06 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:07.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:06 smithi003 ceph-mon[67363]: pgmap v297: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:07.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:06 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:07.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:06 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:07.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:06 smithi027 ceph-mon[53801]: pgmap v297: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:08.282 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:08 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:08.282 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:08 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:08.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:08 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:08.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:08 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:09.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:09 smithi003 ceph-mon[67363]: pgmap v298: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:09.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:09 smithi027 ceph-mon[53801]: pgmap v298: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:11.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: pgmap v299: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:11.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.339 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.340 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.340 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.340 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.340 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.340 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.340 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.341 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-01T02:51:11.342 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-01T02:51:11.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-01T02:51:11.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-01T02:51:11.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-01T02:51:11.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-01T02:51:11.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-01T02:51:11.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-01T02:51:11.343 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-06-01T02:51:11.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-01T02:51:11.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-01T02:51:11.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-01T02:51:11.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-01T02:51:11.344 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:51:11.345 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:51:11.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.346 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:11 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: pgmap v299: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:11.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:51:11.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:51:11.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.354 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-01T02:51:11.355 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-01T02:51:11.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-01T02:51:11.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-01T02:51:11.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-01T02:51:11.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-01T02:51:11.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-01T02:51:11.356 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-01T02:51:11.357 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:51:11.358 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:51:11.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:51:11.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:11.359 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:11 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:12.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:12 smithi003 ceph-mon[67363]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:51:12.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:12 smithi003 ceph-mon[67363]: Upgrade: Finalizing container_image settings 2024-06-01T02:51:12.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:12 smithi003 ceph-mon[67363]: Upgrade: Complete! 2024-06-01T02:51:12.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:51:12.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:51:12.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:51:12.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:12.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:12.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:12 smithi027 ceph-mon[53801]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-01T02:51:12.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:12 smithi027 ceph-mon[53801]: Upgrade: Finalizing container_image settings 2024-06-01T02:51:12.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:12 smithi027 ceph-mon[53801]: Upgrade: Complete! 2024-06-01T02:51:12.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:51:12.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:51:12.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:51:12.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:12.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:51:13.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:13 smithi003 ceph-mon[67363]: pgmap v300: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:13.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:13 smithi027 ceph-mon[53801]: pgmap v300: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:15.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:15 smithi003 ceph-mon[67363]: pgmap v301: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:15.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:15 smithi027 ceph-mon[53801]: pgmap v301: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:16.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:51:16.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:51:17.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:17 smithi027 ceph-mon[53801]: pgmap v302: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:17.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:17 smithi003 ceph-mon[67363]: pgmap v302: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:19.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:19 smithi027 ceph-mon[53801]: pgmap v303: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:19.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:19 smithi003 ceph-mon[67363]: pgmap v303: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:21.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:21 smithi027 ceph-mon[53801]: pgmap v304: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:21.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:21 smithi003 ceph-mon[67363]: pgmap v304: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:23.352 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:23 smithi027 ceph-mon[53801]: pgmap v305: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:23.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:23 smithi003 ceph-mon[67363]: pgmap v305: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:24.353 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph orch ps' 2024-06-01T02:51:25.327 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:25 smithi003 ceph-mon[67363]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:51:25.327 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:25 smithi003 ceph-mon[67363]: pgmap v306: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:25.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:25 smithi027 ceph-mon[53801]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:51:25.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:25 smithi027 ceph-mon[53801]: pgmap v306: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:26.391 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (51s) 20s ago 33m 17.0M - 0.25.0 c8568f914cd2 be8d55fb77d3 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (7m) 20s ago 33m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (7m) 95s ago 32m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (26s) 20s ago 33m 70.3M - 9.4.12 2bacad6d85d8 8ab1c9e17ad9 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (2m) 20s ago 27m 31.6M - 19.0.0-2946-g3d89accf b045ac545f82 00d739c1b4e3 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (2m) 95s ago 27m 12.3M - 19.0.0-2946-g3d89accf b045ac545f82 459ab19de6be 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (9m) 20s ago 34m 524M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (9m) 95s ago 32m 454M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (8m) 20s ago 34m 61.4M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (8m) 95s ago 32m 45.6M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (2m) 20s ago 27m 64.5M - 5.5 b045ac545f82 f0c3e06ef6f3 2024-06-01T02:51:26.392 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (2m) 95s ago 27m 61.2M - 5.5 b045ac545f82 92e8b05a633d 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (105s) 20s ago 33m 7889k - 1.5.0 0da6a335fe13 087d7399c180 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (98s) 95s ago 32m 3497k - 1.5.0 0da6a335fe13 6b2d846d4d14 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (7m) 20s ago 32m 202M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (6m) 20s ago 31m 187M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (6m) 20s ago 31m 157M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (5m) 20s ago 30m 196M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (4m) 95s ago 30m 204M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (4m) 95s ago 29m 196M 2704M 19.0.0-2946-g3d89accf b045ac545f82 cce965464b24 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (3m) 95s ago 29m 189M 2704M 19.0.0-2946-g3d89accf b045ac545f82 13f0723f71f7 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (3m) 95s ago 29m 230M 2704M 19.0.0-2946-g3d89accf b045ac545f82 196fb45013fd 2024-06-01T02:51:26.393 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (73s) 20s ago 32m 53.5M - 2.43.0 a07b618ecd1d fe24a7ed223d 2024-06-01T02:51:27.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:27 smithi003 ceph-mon[67363]: pgmap v307: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:27.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:27 smithi003 ceph-mon[67363]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:51:27.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:27 smithi027 ceph-mon[53801]: pgmap v307: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:27.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:27 smithi027 ceph-mon[53801]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:51:29.482 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:29 smithi003 ceph-mon[67363]: pgmap v308: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:29.514 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph versions' 2024-06-01T02:51:29.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:29 smithi027 ceph-mon[53801]: pgmap v308: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:31.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:31 smithi003 ceph-mon[67363]: pgmap v309: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:31.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 8 2024-06-01T02:51:31.565 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:51:31.566 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:51:31.566 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:51:31.566 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:51:31.566 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:51:31.566 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 14 2024-06-01T02:51:31.566 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:51:31.566 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:51:31.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:31 smithi027 ceph-mon[53801]: pgmap v309: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:31.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:51:32.156 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:32 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/2158913114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:32.157 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-06-01T02:51:32.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:32 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/2158913114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:51:33.376 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:33 smithi003 ceph-mon[67363]: pgmap v310: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:33.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:33 smithi027 ceph-mon[53801]: pgmap v310: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:33.976 INFO:teuthology.orchestra.run.smithi003.stdout:wait for servicemap items w/ changing names to refresh 2024-06-01T02:51:34.566 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'sleep 60' 2024-06-01T02:51:35.207 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:35 smithi003 ceph-mon[67363]: pgmap v311: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:35.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:35 smithi027 ceph-mon[53801]: pgmap v311: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:37.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:37 smithi003 ceph-mon[67363]: pgmap v312: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:37.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:37 smithi027 ceph-mon[53801]: pgmap v312: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:39.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:39 smithi003 ceph-mon[67363]: pgmap v313: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:39.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:39 smithi027 ceph-mon[53801]: pgmap v313: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:41.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:41 smithi003 ceph-mon[67363]: pgmap v314: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:41.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:41 smithi027 ceph-mon[53801]: pgmap v314: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:43.353 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:43 smithi027 ceph-mon[53801]: pgmap v315: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:43.441 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:43 smithi003 ceph-mon[67363]: pgmap v315: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:45.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:45 smithi003 ceph-mon[67363]: pgmap v316: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:45.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:45 smithi027 ceph-mon[53801]: pgmap v316: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:46.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:51:46.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:51:47.504 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:47 smithi027 ceph-mon[53801]: pgmap v317: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:47.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:47 smithi003 ceph-mon[67363]: pgmap v317: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:49.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:49 smithi003 ceph-mon[67363]: pgmap v318: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:49.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:49 smithi027 ceph-mon[53801]: pgmap v318: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:51.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:51 smithi003 ceph-mon[67363]: pgmap v319: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:51.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:51 smithi027 ceph-mon[53801]: pgmap v319: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:53.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:53 smithi003 ceph-mon[67363]: pgmap v320: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:53.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:53 smithi027 ceph-mon[53801]: pgmap v320: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:55.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:55 smithi003 ceph-mon[67363]: pgmap v321: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:55 smithi027 ceph-mon[53801]: pgmap v321: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:57.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:57 smithi003 ceph-mon[67363]: pgmap v322: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:57.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:57 smithi027 ceph-mon[53801]: pgmap v322: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:51:59.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:51:59 smithi003 ceph-mon[67363]: pgmap v323: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:51:59.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:51:59 smithi027 ceph-mon[53801]: pgmap v323: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:01.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:01 smithi003 ceph-mon[67363]: pgmap v324: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:01.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:52:01.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:01 smithi027 ceph-mon[53801]: pgmap v324: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:52:03.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:03 smithi003 ceph-mon[67363]: pgmap v325: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:03.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:03 smithi027 ceph-mon[53801]: pgmap v325: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:05.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:05 smithi003 ceph-mon[67363]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:05.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:05 smithi027 ceph-mon[53801]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:07 smithi003 ceph-mon[67363]: pgmap v327: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:07.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:07 smithi027 ceph-mon[53801]: pgmap v327: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:09.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:09 smithi003 ceph-mon[67363]: pgmap v328: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:09.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:09 smithi027 ceph-mon[53801]: pgmap v328: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:11.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:11 smithi003 ceph-mon[67363]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:11.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:11 smithi027 ceph-mon[53801]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:12.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:52:12.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:52:12.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:52:12.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:52:12.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:12 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:52:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-01T02:52:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-01T02:52:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-01T02:52:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:52:12.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:12 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' 2024-06-01T02:52:13.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:13 smithi003 ceph-mon[67363]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:13.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:13 smithi027 ceph-mon[53801]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:15.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:15 smithi003 ceph-mon[67363]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:15.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:15 smithi027 ceph-mon[53801]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:16.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:16 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:52:16.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:16 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:52:17.504 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:17 smithi027 ceph-mon[53801]: pgmap v332: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:17.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:17 smithi003 ceph-mon[67363]: pgmap v332: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:19.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:19 smithi003 ceph-mon[67363]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:19.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:19 smithi027 ceph-mon[53801]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:21.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:21 smithi003 ceph-mon[67363]: pgmap v334: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:21.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:21 smithi027 ceph-mon[53801]: pgmap v334: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:23.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:23 smithi003 ceph-mon[67363]: pgmap v335: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:23.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:23 smithi027 ceph-mon[53801]: pgmap v335: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:25.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:25 smithi003 ceph-mon[67363]: pgmap v336: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:25.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:25 smithi027 ceph-mon[53801]: pgmap v336: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:27.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:27 smithi003 ceph-mon[67363]: pgmap v337: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:27.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:27 smithi027 ceph-mon[53801]: pgmap v337: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:29.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:29 smithi003 ceph-mon[67363]: pgmap v338: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:29.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:29 smithi027 ceph-mon[53801]: pgmap v338: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:31 smithi003 ceph-mon[67363]: pgmap v339: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:31.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:31 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:52:31.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:31 smithi027 ceph-mon[53801]: pgmap v339: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:31.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:31 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:52:33.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:33 smithi003 ceph-mon[67363]: pgmap v340: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:33.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:33 smithi027 ceph-mon[53801]: pgmap v340: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:35.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:35 smithi003 ceph-mon[67363]: pgmap v341: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:35.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:35 smithi027 ceph-mon[53801]: pgmap v341: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:36.759 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph orch ps' 2024-06-01T02:52:37.333 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:37 smithi003 ceph-mon[67363]: pgmap v342: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:37.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:37 smithi027 ceph-mon[53801]: pgmap v342: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:38.950 INFO:teuthology.orchestra.run.smithi003.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:alertmanager.smithi003 smithi003 *:9093,9094 running (2m) 93s ago 34m 17.0M - 0.25.0 c8568f914cd2 be8d55fb77d3 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi003 smithi003 running (8m) 93s ago 34m 6665k - 19.0.0-2946-g3d89accf b045ac545f82 e202a4e1031e 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:crash.smithi027 smithi027 running (8m) 2m ago 34m 6660k - 19.0.0-2946-g3d89accf b045ac545f82 75a59ecf8739 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:grafana.smithi003 smithi003 *:3000 running (99s) 93s ago 34m 70.3M - 9.4.12 2bacad6d85d8 8ab1c9e17ad9 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi003.kexxec smithi003 running (3m) 93s ago 28m 31.6M - 19.0.0-2946-g3d89accf b045ac545f82 00d739c1b4e3 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:mds.foofs.smithi027.xaykap smithi027 running (3m) 2m ago 28m 12.3M - 19.0.0-2946-g3d89accf b045ac545f82 459ab19de6be 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi003.njsqxz smithi003 *:8443,9283,8765 running (10m) 93s ago 35m 524M - 19.0.0-2946-g3d89accf b045ac545f82 4cbb3c391901 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:mgr.smithi027.tujskj smithi027 *:8443,9283,8765 running (10m) 2m ago 34m 454M - 19.0.0-2946-g3d89accf b045ac545f82 73d93198ea3e 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi003 smithi003 running (10m) 93s ago 36m 61.4M 2048M 19.0.0-2946-g3d89accf b045ac545f82 a3fb82b575ae 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:mon.smithi027 smithi027 running (9m) 2m ago 33m 45.6M 2048M 19.0.0-2946-g3d89accf b045ac545f82 f58a9b5236e4 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.0.0.smithi003.dmybfb smithi003 *:2049 running (3m) 93s ago 28m 64.5M - 5.5 b045ac545f82 f0c3e06ef6f3 2024-06-01T02:52:38.951 INFO:teuthology.orchestra.run.smithi003.stdout:nfs.foo.1.0.smithi027.dsjwox smithi027 *:2049 running (3m) 2m ago 28m 61.2M - 5.5 b045ac545f82 92e8b05a633d 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi003 smithi003 *:9100 running (2m) 93s ago 34m 7889k - 1.5.0 0da6a335fe13 087d7399c180 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:node-exporter.smithi027 smithi027 *:9100 running (2m) 2m ago 33m 3497k - 1.5.0 0da6a335fe13 6b2d846d4d14 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:osd.0 smithi003 running (8m) 93s ago 33m 202M 1937M 19.0.0-2946-g3d89accf b045ac545f82 cfa6fb225bc1 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:osd.1 smithi003 running (7m) 93s ago 32m 187M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2806eedd6c53 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:osd.2 smithi003 running (7m) 93s ago 32m 157M 1937M 19.0.0-2946-g3d89accf b045ac545f82 bc04c16a8783 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:osd.3 smithi003 running (6m) 93s ago 31m 196M 1937M 19.0.0-2946-g3d89accf b045ac545f82 2780cd01c5e6 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:osd.4 smithi027 running (6m) 2m ago 31m 204M 2704M 19.0.0-2946-g3d89accf b045ac545f82 a133545ce0d3 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:osd.5 smithi027 running (5m) 2m ago 31m 196M 2704M 19.0.0-2946-g3d89accf b045ac545f82 cce965464b24 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:osd.6 smithi027 running (4m) 2m ago 30m 189M 2704M 19.0.0-2946-g3d89accf b045ac545f82 13f0723f71f7 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:osd.7 smithi027 running (4m) 2m ago 30m 230M 2704M 19.0.0-2946-g3d89accf b045ac545f82 196fb45013fd 2024-06-01T02:52:38.952 INFO:teuthology.orchestra.run.smithi003.stdout:prometheus.smithi003 smithi003 *:9095 running (2m) 93s ago 33m 53.5M - 2.43.0 a07b618ecd1d fe24a7ed223d 2024-06-01T02:52:39.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:39 smithi003 ceph-mon[67363]: pgmap v343: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:39.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:39 smithi027 ceph-mon[53801]: pgmap v343: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:39.625 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph orch upgrade status' 2024-06-01T02:52:40.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:40 smithi003 ceph-mon[67363]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:52:40.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:40 smithi027 ceph-mon[53801]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:52:41.529 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:52:41.529 INFO:teuthology.orchestra.run.smithi003.stdout: "target_image": null, 2024-06-01T02:52:41.529 INFO:teuthology.orchestra.run.smithi003.stdout: "in_progress": false, 2024-06-01T02:52:41.529 INFO:teuthology.orchestra.run.smithi003.stdout: "which": "", 2024-06-01T02:52:41.529 INFO:teuthology.orchestra.run.smithi003.stdout: "services_complete": [], 2024-06-01T02:52:41.529 INFO:teuthology.orchestra.run.smithi003.stdout: "progress": null, 2024-06-01T02:52:41.529 INFO:teuthology.orchestra.run.smithi003.stdout: "message": "", 2024-06-01T02:52:41.529 INFO:teuthology.orchestra.run.smithi003.stdout: "is_paused": false 2024-06-01T02:52:41.529 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:52:41.530 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:41 smithi003 ceph-mon[67363]: pgmap v344: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:41.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:41 smithi027 ceph-mon[53801]: pgmap v344: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:42.130 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph health detail' 2024-06-01T02:52:42.576 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:42 smithi003 ceph-mon[67363]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:52:42.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:42 smithi027 ceph-mon[53801]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:52:43.410 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:43 smithi003 ceph-mon[67363]: pgmap v345: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:43.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:43 smithi027 ceph-mon[53801]: pgmap v345: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:44.224 INFO:teuthology.orchestra.run.smithi003.stdout:HEALTH_OK 2024-06-01T02:52:44.534 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:44 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/507921380' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:52:44.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:44 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/507921380' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-01T02:52:44.859 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph versions' 2024-06-01T02:52:45.446 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:45 smithi003 ceph-mon[67363]: pgmap v346: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:45.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:45 smithi027 ceph-mon[53801]: pgmap v346: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:46.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:46 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:52:46.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:46 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:52:46.928 INFO:teuthology.orchestra.run.smithi003.stdout:{ 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "mon": { 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "mgr": { 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "osd": { 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 8 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "mds": { 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 2 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "overall": { 2024-06-01T02:52:46.929 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)": 14 2024-06-01T02:52:46.930 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:52:46.930 INFO:teuthology.orchestra.run.smithi003.stdout:} 2024-06-01T02:52:47.515 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:47 smithi003 ceph-mon[67363]: pgmap v347: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:47.515 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:47 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/3402826052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:52:47.541 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-06-01T02:52:47.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:47 smithi027 ceph-mon[53801]: pgmap v347: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:47.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:47 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/3402826052' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:52:49.338 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:49 smithi003 ceph-mon[67363]: pgmap v348: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:49.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:49 smithi027 ceph-mon[53801]: pgmap v348: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:49.615 INFO:teuthology.orchestra.run.smithi003.stdout:true 2024-06-01T02:52:50.321 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:50 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1819985725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:52:50.348 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -e sha1=3d89accfe3201286f000e065a1faea2a88c7a6ba -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-06-01T02:52:50.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:50 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/1819985725' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:52:51.586 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:51 smithi003 ceph-mon[67363]: pgmap v349: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:51.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:51 smithi027 ceph-mon[53801]: pgmap v349: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:52.458 INFO:teuthology.orchestra.run.smithi003.stdout: "ceph version 19.0.0-2946-g3d89accf (3d89accfe3201286f000e065a1faea2a88c7a6ba) squid (rc)" 2024-06-01T02:52:53.056 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-06-01T02:52:53.056 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-06-01T02:52:53.057 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-01T02:52:53.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:53 smithi003 ceph-mon[67363]: pgmap v350: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:53.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:53 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/1329577522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:52:53.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:53 smithi027 ceph-mon[53801]: pgmap v350: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:53.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:53 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/1329577522' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-01T02:52:55.215 INFO:teuthology.orchestra.run.smithi003.stdout: 2024-06-01T02:52:55.215 INFO:teuthology.orchestra.run.smithi003.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-01T02:17:20.272253Z", "last_refresh": "2024-06-01T02:51:05.851663Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-01T02:17:15.344910Z", "last_refresh": "2024-06-01T02:49:50.991666Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-06-01T02:17:17.933675Z", "last_refresh": "2024-06-01T02:51:05.851821Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-01T02:23:42.727611Z", "last_refresh": "2024-06-01T02:49:50.992366Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-01T02:17:13.392361Z", "last_refresh": "2024-06-01T02:49:50.991780Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi003:172.21.15.3=smithi003", "smithi027:172.21.15.27=smithi027"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-01T02:18:03.189384Z", "last_refresh": "2024-06-01T02:49:50.991863Z", "running": 2, "size": 2}}, {"events": ["2024-06-01T02:52:11.895587Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-06-01T02:24:03.347516Z", "last_refresh": "2024-06-01T02:49:50.992444Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-01T02:17:19.190358Z", "last_refresh": "2024-06-01T02:49:50.991956Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9", "last_refresh": "2024-06-01T02:49:50.992046Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-01T02:17:16.680096Z", "last_refresh": "2024-06-01T02:51:05.852000Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-01T02:52:55.524 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:55 smithi003 ceph-mon[67363]: pgmap v351: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:55.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:55 smithi027 ceph-mon[53801]: pgmap v351: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:55.851 INFO:tasks.cephadm:nfs.foo has 2/2 2024-06-01T02:52:55.851 DEBUG:teuthology.parallel:result is None 2024-06-01T02:52:55.852 INFO:teuthology.run_tasks:Running task vip.exec... 2024-06-01T02:52:55.863 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:52:55.863 DEBUG:teuthology.orchestra.run.smithi003:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-06-01T02:52:55.891 INFO:teuthology.orchestra.run.smithi003.stderr:+ umount /mnt/foo 2024-06-01T02:52:55.903 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-01T02:52:55.911 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:52:55.912 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-06-01T02:52:56.546 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:56 smithi003 ceph-mon[67363]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:52:56.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:56 smithi027 ceph-mon[53801]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-01T02:52:57.529 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:57 smithi003 ceph-mon[67363]: pgmap v352: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:57.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:57 smithi027 ceph-mon[53801]: pgmap v352: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:52:57.915 INFO:teuthology.orchestra.run.smithi003.stdout: "foo" 2024-06-01T02:52:58.612 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-06-01T02:52:59.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:59 smithi003 ceph-mon[67363]: pgmap v353: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:59.588 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:52:59 smithi003 ceph-mon[67363]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:52:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:59 smithi027 ceph-mon[53801]: pgmap v353: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-01T02:52:59.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:52:59 smithi027 ceph-mon[53801]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:53:00.752 INFO:teuthology.orchestra.run.smithi003.stdout:[ 2024-06-01T02:53:00.752 INFO:teuthology.orchestra.run.smithi003.stdout: { 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "access_type": "RW", 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "clients": [], 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "cluster_id": "foo", 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "export_id": 1, 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "fsal": { 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "fs_name": "foofs", 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "name": "CEPH", 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "user_id": "nfs.foo.1" 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: }, 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "path": "/", 2024-06-01T02:53:00.753 INFO:teuthology.orchestra.run.smithi003.stdout: "protocols": [ 2024-06-01T02:53:00.754 INFO:teuthology.orchestra.run.smithi003.stdout: 4 2024-06-01T02:53:00.754 INFO:teuthology.orchestra.run.smithi003.stdout: ], 2024-06-01T02:53:00.754 INFO:teuthology.orchestra.run.smithi003.stdout: "pseudo": "/fake", 2024-06-01T02:53:00.754 INFO:teuthology.orchestra.run.smithi003.stdout: "security_label": true, 2024-06-01T02:53:00.754 INFO:teuthology.orchestra.run.smithi003.stdout: "squash": "none", 2024-06-01T02:53:00.754 INFO:teuthology.orchestra.run.smithi003.stdout: "transports": [ 2024-06-01T02:53:00.754 INFO:teuthology.orchestra.run.smithi003.stdout: "TCP" 2024-06-01T02:53:00.754 INFO:teuthology.orchestra.run.smithi003.stdout: ] 2024-06-01T02:53:00.754 INFO:teuthology.orchestra.run.smithi003.stdout: } 2024-06-01T02:53:00.755 INFO:teuthology.orchestra.run.smithi003.stdout:] 2024-06-01T02:53:01.390 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:01 smithi003 ceph-mon[67363]: pgmap v354: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:53:01.390 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:01 smithi003 ceph-mon[67363]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:53:01.416 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-06-01T02:53:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:01 smithi027 ceph-mon[53801]: pgmap v354: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:53:01.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:01 smithi027 ceph-mon[53801]: from='mgr.34104 172.21.15.3:0/1416358393' entity='mgr.smithi003.njsqxz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-01T02:53:02.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:02 smithi003 ceph-mon[67363]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:53:02.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:02 smithi027 ceph-mon[53801]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-01T02:53:03.166 INFO:teuthology.orchestra.run.smithi003.stdout:foo rec-0000000000000004:nfs.foo.1 2024-06-01T02:53:03.167 INFO:teuthology.orchestra.run.smithi003.stdout:foo grace 2024-06-01T02:53:03.167 INFO:teuthology.orchestra.run.smithi003.stdout:foo rec-0000000000000004:nfs.foo.0 2024-06-01T02:53:03.167 INFO:teuthology.orchestra.run.smithi003.stdout:foo conf-nfs.foo 2024-06-01T02:53:03.167 INFO:teuthology.orchestra.run.smithi003.stdout:foo export-1 2024-06-01T02:53:03.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:03 smithi003 ceph-mon[67363]: pgmap v355: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:53:03.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:03 smithi003 ceph-mon[67363]: mgrmap e42: smithi003.njsqxz(active, since 10m), standbys: smithi027.tujskj 2024-06-01T02:53:03.602 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:03 smithi027 ceph-mon[53801]: pgmap v355: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-01T02:53:03.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:03 smithi027 ceph-mon[53801]: mgrmap e42: smithi003.njsqxz(active, since 10m), standbys: smithi027.tujskj 2024-06-01T02:53:03.820 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-01T02:53:03.829 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:53:03.829 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 -- bash -c 'set -ex 2024-06-01T02:53:03.830 DEBUG:teuthology.orchestra.run.smithi003:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-06-01T02:53:03.830 DEBUG:teuthology.orchestra.run.smithi003:> ' 2024-06-01T02:53:05.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:05 smithi003 ceph-mon[67363]: pgmap v356: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:53:05.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:05 smithi027 ceph-mon[53801]: pgmap v356: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-01T02:53:05.814 INFO:teuthology.orchestra.run.smithi003.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-06-01T02:53:06.078 INFO:teuthology.orchestra.run.smithi003.stderr:+ [[ 6 -gt 2 ]] 2024-06-01T02:53:06.337 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:06 smithi003 ceph-mon[67363]: from='client.? 172.21.15.3:0/2433037875' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-06-01T02:53:06.603 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:06 smithi027 ceph-mon[53801]: from='client.? 172.21.15.3:0/2433037875' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-06-01T02:53:06.661 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-06-01T02:53:06.671 INFO:tasks.cephadm:Teardown begin 2024-06-01T02:53:06.672 DEBUG:teuthology.orchestra.run.smithi003:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-01T02:53:06.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-01T02:53:06.735 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-06-01T02:53:06.736 DEBUG:teuthology.orchestra.run.smithi003:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-06-01T02:53:06.757 DEBUG:teuthology.orchestra.run.smithi027:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-06-01T02:53:06.795 INFO:tasks.cephadm:Stopping all daemons... 2024-06-01T02:53:06.795 INFO:tasks.cephadm.mon.smithi003:Stopping mon.smithi003... 2024-06-01T02:53:06.795 DEBUG:teuthology.orchestra.run.smithi003:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi003 2024-06-01T02:53:07.087 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:06 smithi003 systemd[1]: Stopping Ceph mon.smithi003 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:53:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:07 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003[67359]: 2024-06-01T02:53:07.123+0000 7fa41aaea700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi003 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:53:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:07 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003[67359]: 2024-06-01T02:53:07.123+0000 7fa41aaea700 -1 mon.smithi003@0(leader) e3 *** Got Signal Terminated *** 2024-06-01T02:53:07.587 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:07 smithi003 podman[112836]: 2024-06-01 02:53:07.359403596 +0000 UTC m=+0.368182332 container died a3fb82b575ae1f33e56589242e4ea7b3134b5aadc6f38d9cd72649b1b2583a5d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-06-01T02:53:08.007 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:07 smithi003 podman[112836]: 2024-06-01 02:53:07.739153267 +0000 UTC m=+0.747932009 container cleanup a3fb82b575ae1f33e56589242e4ea7b3134b5aadc6f38d9cd72649b1b2583a5d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc) 2024-06-01T02:53:08.007 INFO:journalctl@ceph.mon.smithi003.smithi003.stdout:Jun 01 02:53:07 smithi003 bash[112836]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi003 2024-06-01T02:53:08.136 DEBUG:teuthology.orchestra.run.smithi003:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi003.service' 2024-06-01T02:53:08.185 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:53:08.185 INFO:tasks.cephadm.mon.smithi003:Stopped mon.smithi003 2024-06-01T02:53:08.185 INFO:tasks.cephadm.mon.smithi027:Stopping mon.smithi027... 2024-06-01T02:53:08.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi027 2024-06-01T02:53:08.486 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:08 smithi027 systemd[1]: Stopping Ceph mon.smithi027 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:53:08.828 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027[53797]: 2024-06-01T02:53:08.483+0000 7f1f10296700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi027 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:53:08.828 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027[53797]: 2024-06-01T02:53:08.483+0000 7f1f10296700 -1 mon.smithi027@1(peon) e3 *** Got Signal Terminated *** 2024-06-01T02:53:08.828 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:08 smithi027 podman[79734]: 2024-06-01 02:53:08.556893318 +0000 UTC m=+0.181329079 container died f58a9b5236e40de221ae0ff0acb4b020858c929726a6ddfd6d70cff9e36f9778 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:53:09.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:09 smithi027 podman[79734]: 2024-06-01 02:53:09.010517243 +0000 UTC m=+0.634953003 container cleanup f58a9b5236e40de221ae0ff0acb4b020858c929726a6ddfd6d70cff9e36f9778 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:53:09.103 INFO:journalctl@ceph.mon.smithi027.smithi027.stdout:Jun 01 02:53:09 smithi027 bash[79734]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-mon-smithi027 2024-06-01T02:53:09.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@mon.smithi027.service' 2024-06-01T02:53:09.496 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:53:09.497 INFO:tasks.cephadm.mon.smithi027:Stopped mon.smithi027 2024-06-01T02:53:09.497 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-06-01T02:53:09.497 DEBUG:teuthology.orchestra.run.smithi003:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.0 2024-06-01T02:53:09.810 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:09 smithi003 systemd[1]: Stopping Ceph osd.0 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:53:10.087 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:09 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[73431]: 2024-06-01T02:53:09.809+0000 7f27fa4ea700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:53:10.087 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:09 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[73431]: 2024-06-01T02:53:09.809+0000 7f27fa4ea700 -1 osd.0 130 *** Got signal Terminated *** 2024-06-01T02:53:10.088 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:09 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0[73431]: 2024-06-01T02:53:09.809+0000 7f27fa4ea700 -1 osd.0 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:53:15.587 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:15 smithi003 podman[112967]: 2024-06-01 02:53:15.298542623 +0000 UTC m=+5.591926227 container died cfa6fb225bc1a4102c6cc321e63d9d0aac8e87419a17a0f1131f697835e23672 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-06-01T02:53:16.062 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:15 smithi003 podman[112967]: 2024-06-01 02:53:15.79330641 +0000 UTC m=+6.086690013 container cleanup cfa6fb225bc1a4102c6cc321e63d9d0aac8e87419a17a0f1131f697835e23672 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-06-01T02:53:16.062 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:15 smithi003 bash[112967]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0 2024-06-01T02:53:16.337 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:16 smithi003 podman[112982]: 2024-06-01 02:53:16.06174501 +0000 UTC m=+0.765836923 container remove cfa6fb225bc1a4102c6cc321e63d9d0aac8e87419a17a0f1131f697835e23672 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-06-01T02:53:16.773 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:16 smithi003 podman[113071]: 2024-06-01 02:53:16.345067128 +0000 UTC m=+0.018272558 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:53:16.773 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:16 smithi003 podman[113071]: 2024-06-01 02:53:16.446010159 +0000 UTC m=+0.119215572 container create f57cb20a5875cb02f192a9c65effda1fd0422ce2f92170bee69600b1a981f68d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux ) 2024-06-01T02:53:16.773 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:16 smithi003 podman[113071]: 2024-06-01 02:53:16.681878659 +0000 UTC m=+0.355084076 container init f57cb20a5875cb02f192a9c65effda1fd0422ce2f92170bee69600b1a981f68d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-06-01T02:53:16.774 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:16 smithi003 podman[113071]: 2024-06-01 02:53:16.685700803 +0000 UTC m=+0.358906225 container start f57cb20a5875cb02f192a9c65effda1fd0422ce2f92170bee69600b1a981f68d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:53:16.774 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:16 smithi003 podman[113071]: 2024-06-01 02:53:16.773614833 +0000 UTC m=+0.446820256 container attach f57cb20a5875cb02f192a9c65effda1fd0422ce2f92170bee69600b1a981f68d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528) 2024-06-01T02:53:17.087 INFO:journalctl@ceph.osd.0.smithi003.stdout:Jun 01 02:53:16 smithi003 podman[113071]: 2024-06-01 02:53:16.994943387 +0000 UTC m=+0.668148810 container died f57cb20a5875cb02f192a9c65effda1fd0422ce2f92170bee69600b1a981f68d (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240528) 2024-06-01T02:53:17.545 DEBUG:teuthology.orchestra.run.smithi003:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.0.service' 2024-06-01T02:53:17.585 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:53:17.586 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-06-01T02:53:17.586 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-06-01T02:53:17.586 DEBUG:teuthology.orchestra.run.smithi003:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.1 2024-06-01T02:53:17.837 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:17 smithi003 systemd[1]: Stopping Ceph osd.1 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:53:18.233 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:17 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:53:17.964+0000 7f63c273c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:53:18.233 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:17 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:53:17.964+0000 7f63c273c700 -1 osd.1 130 *** Got signal Terminated *** 2024-06-01T02:53:18.233 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:17 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1[76072]: 2024-06-01T02:53:17.964+0000 7f63c273c700 -1 osd.1 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:53:23.623 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:23 smithi003 podman[113186]: 2024-06-01 02:53:23.362190903 +0000 UTC m=+5.527635200 container died 2806eedd6c53f0c3e62dc00eaaf7febaf00998ed606ecd9b60ab50b8a6841415 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-06-01T02:53:24.006 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:23 smithi003 podman[113186]: 2024-06-01 02:53:23.730595312 +0000 UTC m=+5.896039604 container cleanup 2806eedd6c53f0c3e62dc00eaaf7febaf00998ed606ecd9b60ab50b8a6841415 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:53:24.007 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:23 smithi003 bash[113186]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1 2024-06-01T02:53:24.337 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:24 smithi003 podman[113236]: 2024-06-01 02:53:24.006899672 +0000 UTC m=+0.642499028 container remove 2806eedd6c53f0c3e62dc00eaaf7febaf00998ed606ecd9b60ab50b8a6841415 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:53:24.694 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:24 smithi003 podman[113323]: 2024-06-01 02:53:24.36627099 +0000 UTC m=+0.107738795 container create db76de2c59f606927dc6edb077ae77178138c0ec1261b2f3c45da74db5684451 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-06-01T02:53:24.694 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:24 smithi003 podman[113323]: 2024-06-01 02:53:24.277578338 +0000 UTC m=+0.019046105 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:53:24.694 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:24 smithi003 podman[113323]: 2024-06-01 02:53:24.602233273 +0000 UTC m=+0.343701026 container init db76de2c59f606927dc6edb077ae77178138c0ec1261b2f3c45da74db5684451 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:53:24.694 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:24 smithi003 podman[113323]: 2024-06-01 02:53:24.605616848 +0000 UTC m=+0.347084609 container start db76de2c59f606927dc6edb077ae77178138c0ec1261b2f3c45da74db5684451 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8) 2024-06-01T02:53:24.694 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:24 smithi003 podman[113323]: 2024-06-01 02:53:24.693916099 +0000 UTC m=+0.435383871 container attach db76de2c59f606927dc6edb077ae77178138c0ec1261b2f3c45da74db5684451 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc) 2024-06-01T02:53:25.087 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:24 smithi003 podman[113323]: 2024-06-01 02:53:24.912144779 +0000 UTC m=+0.653612541 container died db76de2c59f606927dc6edb077ae77178138c0ec1261b2f3c45da74db5684451 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, ceph=True, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-06-01T02:53:25.448 INFO:journalctl@ceph.osd.1.smithi003.stdout:Jun 01 02:53:25 smithi003 podman[113349]: 2024-06-01 02:53:25.430508673 +0000 UTC m=+0.510465966 container remove db76de2c59f606927dc6edb077ae77178138c0ec1261b2f3c45da74db5684451 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-06-01T02:53:25.452 DEBUG:teuthology.orchestra.run.smithi003:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.1.service' 2024-06-01T02:53:25.496 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:53:25.497 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-06-01T02:53:25.497 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-06-01T02:53:25.497 DEBUG:teuthology.orchestra.run.smithi003:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.2 2024-06-01T02:53:25.837 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:25 smithi003 systemd[1]: Stopping Ceph osd.2 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:53:26.337 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:25 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[78703]: 2024-06-01T02:53:25.854+0000 7ff74e818700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:53:26.337 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:25 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[78703]: 2024-06-01T02:53:25.854+0000 7ff74e818700 -1 osd.2 130 *** Got signal Terminated *** 2024-06-01T02:53:26.337 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:25 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2[78703]: 2024-06-01T02:53:25.854+0000 7ff74e818700 -1 osd.2 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:53:31.660 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:31 smithi003 podman[113427]: 2024-06-01 02:53:31.37351169 +0000 UTC m=+5.638333316 container died bc04c16a878399eb5d45019099021d9e94840cc5e2f8936d3280fd7226d05eab (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-06-01T02:53:32.087 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:31 smithi003 podman[113427]: 2024-06-01 02:53:31.858563533 +0000 UTC m=+6.123385159 container cleanup bc04c16a878399eb5d45019099021d9e94840cc5e2f8936d3280fd7226d05eab (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528) 2024-06-01T02:53:32.087 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:31 smithi003 bash[113427]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2 2024-06-01T02:53:32.505 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:32 smithi003 podman[113446]: 2024-06-01 02:53:32.143863917 +0000 UTC m=+0.763828098 container remove bc04c16a878399eb5d45019099021d9e94840cc5e2f8936d3280fd7226d05eab (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-01T02:53:32.816 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:32 smithi003 podman[113530]: 2024-06-01 02:53:32.40853202 +0000 UTC m=+0.018545986 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:53:32.816 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:32 smithi003 podman[113530]: 2024-06-01 02:53:32.529229138 +0000 UTC m=+0.139243078 container create 1b98909dfcc608210f0834af7a5ba79ab0528de3759997c8e812b78baa8aa0c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-06-01T02:53:32.816 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:32 smithi003 podman[113530]: 2024-06-01 02:53:32.730426322 +0000 UTC m=+0.340440265 container init 1b98909dfcc608210f0834af7a5ba79ab0528de3759997c8e812b78baa8aa0c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-06-01T02:53:32.816 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:32 smithi003 podman[113530]: 2024-06-01 02:53:32.733793761 +0000 UTC m=+0.343807720 container start 1b98909dfcc608210f0834af7a5ba79ab0528de3759997c8e812b78baa8aa0c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-06-01T02:53:32.817 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:32 smithi003 podman[113530]: 2024-06-01 02:53:32.813717455 +0000 UTC m=+0.423731406 container attach 1b98909dfcc608210f0834af7a5ba79ab0528de3759997c8e812b78baa8aa0c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux ) 2024-06-01T02:53:33.064 INFO:journalctl@ceph.osd.2.smithi003.stdout:Jun 01 02:53:33 smithi003 podman[113530]: 2024-06-01 02:53:33.042656575 +0000 UTC m=+0.652670533 container died 1b98909dfcc608210f0834af7a5ba79ab0528de3759997c8e812b78baa8aa0c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-2-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:53:33.713 DEBUG:teuthology.orchestra.run.smithi003:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.2.service' 2024-06-01T02:53:33.753 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:53:33.754 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-06-01T02:53:33.754 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-06-01T02:53:33.754 DEBUG:teuthology.orchestra.run.smithi003:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.3 2024-06-01T02:53:34.087 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:33 smithi003 systemd[1]: Stopping Ceph osd.3 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:53:34.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:34 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:53:34.129+0000 7fb2f9412700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:53:34.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:34 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:53:34.129+0000 7fb2f9412700 -1 osd.3 130 *** Got signal Terminated *** 2024-06-01T02:53:34.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:34 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:53:34.129+0000 7fb2f9412700 -1 osd.3 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:53:36.034 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:35 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:35.734+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:36.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:36 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:36.033+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:36.837 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:36 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:53:36.374+0000 7fb2f5a06700 -1 osd.3 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:12.659022+0000 front 2024-06-01T02:53:12.659168+0000 (oldest deadline 2024-06-01T02:53:35.559057+0000) 2024-06-01T02:53:37.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:37 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:37.023+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:37.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:36 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:36.778+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:37.837 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:37 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:53:37.397+0000 7fb2f5a06700 -1 osd.3 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:12.659022+0000 front 2024-06-01T02:53:12.659168+0000 (oldest deadline 2024-06-01T02:53:35.559057+0000) 2024-06-01T02:53:38.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:37 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:37.991+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:38.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:37 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:37.773+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:38.837 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:38 smithi003 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3[81337]: 2024-06-01T02:53:38.348+0000 7fb2f5a06700 -1 osd.3 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:12.659022+0000 front 2024-06-01T02:53:12.659168+0000 (oldest deadline 2024-06-01T02:53:35.559057+0000) 2024-06-01T02:53:39.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:38 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:38.950+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:39.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:38 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:38.742+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:39.104 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:38 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:38.943+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:39.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:39 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:39.171+0000 7f88b4de1700 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.170266+0000 front 2024-06-01T02:53:11.170391+0000 (oldest deadline 2024-06-01T02:53:38.770968+0000) 2024-06-01T02:53:39.933 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:39 smithi003 podman[113647]: 2024-06-01 02:53:39.678322704 +0000 UTC m=+5.692904550 container died 2780cd01c5e61d87f8a9ae602751ab37de65c01b6e7954a7d7d14a1b93ca3ae2 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-3d89acc) 2024-06-01T02:53:40.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:39 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:39.942+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:40.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:39 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:39.792+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:40.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:39 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:39.984+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:40.188 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:40 smithi003 podman[113647]: 2024-06-01 02:53:40.003924639 +0000 UTC m=+6.018506477 container cleanup 2780cd01c5e61d87f8a9ae602751ab37de65c01b6e7954a7d7d14a1b93ca3ae2 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-06-01T02:53:40.188 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:40 smithi003 bash[113647]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3 2024-06-01T02:53:40.587 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:40 smithi003 podman[113695]: 2024-06-01 02:53:40.188253876 +0000 UTC m=+0.508862937 container remove 2780cd01c5e61d87f8a9ae602751ab37de65c01b6e7954a7d7d14a1b93ca3ae2 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-06-01T02:53:40.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:40 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:40.189+0000 7f88b4de1700 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.170266+0000 front 2024-06-01T02:53:11.170391+0000 (oldest deadline 2024-06-01T02:53:38.770968+0000) 2024-06-01T02:53:40.983 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:40 smithi003 podman[113784]: 2024-06-01 02:53:40.544665286 +0000 UTC m=+0.018465072 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:53:40.983 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:40 smithi003 podman[113784]: 2024-06-01 02:53:40.64753929 +0000 UTC m=+0.121339097 container create b4ec4729081ddf2388d2c28cdee57ce9a12afca5694615a8a171fb52f79039b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.build-date=20240528, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc) 2024-06-01T02:53:40.983 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:40 smithi003 podman[113784]: 2024-06-01 02:53:40.875145053 +0000 UTC m=+0.348944827 container init b4ec4729081ddf2388d2c28cdee57ce9a12afca5694615a8a171fb52f79039b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, ceph=True, GIT_BRANCH=HEAD) 2024-06-01T02:53:40.984 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:40 smithi003 podman[113784]: 2024-06-01 02:53:40.878684563 +0000 UTC m=+0.352484329 container start b4ec4729081ddf2388d2c28cdee57ce9a12afca5694615a8a171fb52f79039b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-3d89acc, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, ceph=True) 2024-06-01T02:53:40.984 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:40 smithi003 podman[113784]: 2024-06-01 02:53:40.98344476 +0000 UTC m=+0.457244556 container attach b4ec4729081ddf2388d2c28cdee57ce9a12afca5694615a8a171fb52f79039b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:53:41.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:40 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:40.760+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:41.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:40 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:40.980+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:41.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:40 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:40.964+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:41.337 INFO:journalctl@ceph.osd.3.smithi003.stdout:Jun 01 02:53:41 smithi003 podman[113784]: 2024-06-01 02:53:41.184476059 +0000 UTC m=+0.658275827 container died b4ec4729081ddf2388d2c28cdee57ce9a12afca5694615a8a171fb52f79039b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-3-deactivate, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True) 2024-06-01T02:53:41.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:41 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:41.201+0000 7f88b4de1700 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.170266+0000 front 2024-06-01T02:53:11.170391+0000 (oldest deadline 2024-06-01T02:53:38.770968+0000) 2024-06-01T02:53:41.770 DEBUG:teuthology.orchestra.run.smithi003:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.3.service' 2024-06-01T02:53:41.811 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:53:41.811 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-06-01T02:53:41.811 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-06-01T02:53:41.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.4 2024-06-01T02:53:41.972 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:41 smithi027 systemd[1]: Stopping Ceph osd.4 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:53:41.973 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:41 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:41.956+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:41.973 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:41 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:41.718+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:42.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:41 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:41.972+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:42.353 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:42 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:42.155+0000 7f88bc7f5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:53:42.353 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:42 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:42.155+0000 7f88bc7f5700 -1 osd.4 130 *** Got signal Terminated *** 2024-06-01T02:53:42.353 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:42 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:42.155+0000 7f88bc7f5700 -1 osd.4 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:53:42.353 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:42 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:42.168+0000 7f88b4de1700 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.170266+0000 front 2024-06-01T02:53:11.170391+0000 (oldest deadline 2024-06-01T02:53:38.770968+0000) 2024-06-01T02:53:42.944 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:42 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:42.672+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:43.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:42 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:42.981+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:43.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:42 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:42.943+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:43.353 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:43 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:43.162+0000 7f88b4de1700 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.170266+0000 front 2024-06-01T02:53:11.170391+0000 (oldest deadline 2024-06-01T02:53:38.770968+0000) 2024-06-01T02:53:43.926 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:43 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:43.654+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:44.186 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:44 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:44.010+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:44.186 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:43 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:43.926+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:44.187 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:43 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:43.926+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:44.603 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:44 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:44.186+0000 7f88b4de1700 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.170266+0000 front 2024-06-01T02:53:11.170391+0000 (oldest deadline 2024-06-01T02:53:38.770968+0000) 2024-06-01T02:53:44.903 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:44 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:44.647+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:45.353 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:45.135+0000 7f88b4de1700 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.170266+0000 front 2024-06-01T02:53:11.170391+0000 (oldest deadline 2024-06-01T02:53:38.770968+0000) 2024-06-01T02:53:45.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:44 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:44.995+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:45.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:44 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:44.902+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:45.354 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:44 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:44.902+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:45.904 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:45.636+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:46.353 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:46 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:46.139+0000 7f88b4de1700 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.170266+0000 front 2024-06-01T02:53:11.170391+0000 (oldest deadline 2024-06-01T02:53:38.770968+0000) 2024-06-01T02:53:46.353 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:46 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4[58590]: 2024-06-01T02:53:46.139+0000 7f88b4de1700 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.871949+0000 front 2024-06-01T02:53:22.872018+0000 (oldest deadline 2024-06-01T02:53:45.771871+0000) 2024-06-01T02:53:46.354 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:45.959+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:46.354 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:45.903+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:46.354 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:45 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:45.903+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:46.931 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:46 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:46.635+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:47.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:46 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:46.929+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:47.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:46 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:46.929+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:47.354 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:46 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:46.929+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:47.897 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:47 smithi027 podman[79918]: 2024-06-01 02:53:47.678249048 +0000 UTC m=+5.659921195 container died a133545ce0d3b1a05882a17a41d32e9dbbddc2b121b3a12e8ca86128b035b306 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True) 2024-06-01T02:53:47.897 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:47 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:47.676+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:47.897 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:47 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:47.676+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:48.189 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:47 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:47.895+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:48.189 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:47 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:47.895+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:48.189 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:47 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:47.925+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:48.189 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:47 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:47.925+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:48.491 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:48 smithi027 podman[79918]: 2024-06-01 02:53:48.189058512 +0000 UTC m=+6.170730647 container cleanup a133545ce0d3b1a05882a17a41d32e9dbbddc2b121b3a12e8ca86128b035b306 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-3d89acc, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-06-01T02:53:48.491 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:48 smithi027 bash[79918]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4 2024-06-01T02:53:48.853 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:48 smithi027 podman[79934]: 2024-06-01 02:53:48.49074361 +0000 UTC m=+0.805965311 container remove a133545ce0d3b1a05882a17a41d32e9dbbddc2b121b3a12e8ca86128b035b306 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-3d89acc, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-06-01T02:53:48.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:48 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:48.686+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:48.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:48 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:48.686+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:49.236 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:48 smithi027 podman[80023]: 2024-06-01 02:53:48.87963769 +0000 UTC m=+0.017799726 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:53:49.236 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:49 smithi027 podman[80023]: 2024-06-01 02:53:49.000119382 +0000 UTC m=+0.138281474 container create c7d25763c26d050ca96827e1837656cac5f8051cdefe008fa627e91c34d97d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-06-01T02:53:49.237 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:48 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:48.914+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:49.237 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:48 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:48.914+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:49.237 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:48 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:48.945+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:49.237 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:48 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:48.945+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:49.549 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:49 smithi027 podman[80023]: 2024-06-01 02:53:49.236509595 +0000 UTC m=+0.374671634 container init c7d25763c26d050ca96827e1837656cac5f8051cdefe008fa627e91c34d97d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-06-01T02:53:49.550 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:49 smithi027 podman[80023]: 2024-06-01 02:53:49.241005457 +0000 UTC m=+0.379167477 container start c7d25763c26d050ca96827e1837656cac5f8051cdefe008fa627e91c34d97d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-3d89acc) 2024-06-01T02:53:49.550 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:49 smithi027 podman[80023]: 2024-06-01 02:53:49.352794344 +0000 UTC m=+0.490956363 container attach c7d25763c26d050ca96827e1837656cac5f8051cdefe008fa627e91c34d97d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, ceph=True, org.label-schema.build-date=20240528, io.buildah.version=1.33.5) 2024-06-01T02:53:49.550 INFO:journalctl@ceph.osd.4.smithi027.stdout:Jun 01 02:53:49 smithi027 podman[80023]: 2024-06-01 02:53:49.549718571 +0000 UTC m=+0.687880616 container died c7d25763c26d050ca96827e1837656cac5f8051cdefe008fa627e91c34d97d79 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True) 2024-06-01T02:53:49.831 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:49 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:49.658+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:49.832 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:49 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:49.658+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:50.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:49 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:49.990+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:50.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:49 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:49.990+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:50.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:49 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:49.962+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:50.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:49 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:49.962+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:50.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.4.service' 2024-06-01T02:53:50.165 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:53:50.165 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-06-01T02:53:50.165 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-06-01T02:53:50.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.5 2024-06-01T02:53:50.470 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:50 smithi027 systemd[1]: Stopping Ceph osd.5 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:53:50.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:50 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:50.644+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:50.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:50 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:50.644+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:50.853 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:50 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:50.534+0000 7effa2e6e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:53:50.854 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:50 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:50.534+0000 7effa2e6e700 -1 osd.5 130 *** Got signal Terminated *** 2024-06-01T02:53:50.854 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:50 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:50.534+0000 7effa2e6e700 -1 osd.5 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:53:51.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:50 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:50.919+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:51.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:50 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:50.919+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:51.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:50 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:50.975+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:51.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:50 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:50.975+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:51.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:51 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:51.597+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:51.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:51 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:51.597+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:51.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:51 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:51.597+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:53:52.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:51 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:51.992+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:52.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:51 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:51.992+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:52.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:51 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:51.913+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:52.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:51 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:51.913+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:52.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:51 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:51.913+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:25.730562+0000 front 2024-06-01T02:53:25.730447+0000 (oldest deadline 2024-06-01T02:53:51.530804+0000) 2024-06-01T02:53:52.935 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:52.634+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:52.936 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:52.634+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:52.936 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:52.634+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:53:53.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:52.948+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:53.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:52.948+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:53.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:52.948+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:53:53.354 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:52.935+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:53.354 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:52.935+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:53.354 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:52 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:52.935+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:25.730562+0000 front 2024-06-01T02:53:25.730447+0000 (oldest deadline 2024-06-01T02:53:51.530804+0000) 2024-06-01T02:53:53.965 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:53.627+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:53.965 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:53.627+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:53.965 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:53.627+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:53:54.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:53.963+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:54.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:53.963+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:54.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:53.963+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:25.730562+0000 front 2024-06-01T02:53:25.730447+0000 (oldest deadline 2024-06-01T02:53:51.530804+0000) 2024-06-01T02:53:54.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:53.982+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:54.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:53.982+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:54.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:53 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:53.982+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:53:54.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:54 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:54.600+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:54.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:54 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:54.600+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:54.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:54 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:54.600+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:53:55.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:55 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:55.014+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.228157+0000 front 2024-06-01T02:53:11.228362+0000 (oldest deadline 2024-06-01T02:53:35.328157+0000) 2024-06-01T02:53:55.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:55 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:55.014+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:20.029179+0000 front 2024-06-01T02:53:20.029230+0000 (oldest deadline 2024-06-01T02:53:43.529283+0000) 2024-06-01T02:53:55.353 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:55 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5[60846]: 2024-06-01T02:53:55.014+0000 7eff9b45a700 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:25.730562+0000 front 2024-06-01T02:53:25.730447+0000 (oldest deadline 2024-06-01T02:53:51.530804+0000) 2024-06-01T02:53:55.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:54 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:54.957+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:55.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:54 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:54.957+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:55.354 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:54 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:54.957+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:53:55.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:55 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:55.563+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:55.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:55 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:55.563+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:55.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:55 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:55.563+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:53:56.311 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:56 smithi027 podman[80139]: 2024-06-01 02:53:56.061382295 +0000 UTC m=+5.640460524 container died cce965464b24de85407a228e021894f5b73c0a6cfdc7af6b96e63ebf8652455f (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=) 2024-06-01T02:53:56.311 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:55 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:55.934+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:56.311 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:55 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:55.934+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:56.311 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:55 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:55.934+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:53:56.853 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:56 smithi027 podman[80139]: 2024-06-01 02:53:56.61859738 +0000 UTC m=+6.197675608 container cleanup cce965464b24de85407a228e021894f5b73c0a6cfdc7af6b96e63ebf8652455f (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True) 2024-06-01T02:53:56.853 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:56 smithi027 bash[80139]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5 2024-06-01T02:53:56.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:56 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:56.585+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:56.854 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:56 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:56.585+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:56.854 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:56 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:56.585+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:53:57.211 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:56 smithi027 podman[80155]: 2024-06-01 02:53:56.878473855 +0000 UTC m=+0.815072804 container remove cce965464b24de85407a228e021894f5b73c0a6cfdc7af6b96e63ebf8652455f (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:53:57.211 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:56 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:56.922+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:57.212 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:56 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:56.922+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:57.212 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:56 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:56.922+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:53:57.474 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:57 smithi027 podman[80245]: 2024-06-01 02:53:57.114148509 +0000 UTC m=+0.017915124 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:53:57.475 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:57 smithi027 podman[80245]: 2024-06-01 02:53:57.229272826 +0000 UTC m=+0.133039408 container create 62b6da2377df8b3abee86e44898eeb0fc1996f3e18d924080fe40f44ab4b37fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-06-01T02:53:57.475 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:57 smithi027 podman[80245]: 2024-06-01 02:53:57.474443011 +0000 UTC m=+0.378209608 container init 62b6da2377df8b3abee86e44898eeb0fc1996f3e18d924080fe40f44ab4b37fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-3d89acc, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-06-01T02:53:57.788 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:57 smithi027 podman[80245]: 2024-06-01 02:53:57.47857322 +0000 UTC m=+0.382339827 container start 62b6da2377df8b3abee86e44898eeb0fc1996f3e18d924080fe40f44ab4b37fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-06-01T02:53:57.788 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:57 smithi027 podman[80245]: 2024-06-01 02:53:57.557356191 +0000 UTC m=+0.461122788 container attach 62b6da2377df8b3abee86e44898eeb0fc1996f3e18d924080fe40f44ab4b37fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240528, RELEASE=squid-3d89acc, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-06-01T02:53:57.788 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:57 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:57.592+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:57.788 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:57 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:57.592+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:57.788 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:57 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:57.592+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:53:58.103 INFO:journalctl@ceph.osd.5.smithi027.stdout:Jun 01 02:53:57 smithi027 podman[80245]: 2024-06-01 02:53:57.78854156 +0000 UTC m=+0.692308155 container died 62b6da2377df8b3abee86e44898eeb0fc1996f3e18d924080fe40f44ab4b37fd (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240528) 2024-06-01T02:53:58.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:57 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:57.879+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:58.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:57 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:57.879+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:58.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:57 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:57.879+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:53:58.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.5.service' 2024-06-01T02:53:58.477 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:53:58.477 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-06-01T02:53:58.477 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-06-01T02:53:58.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.6 2024-06-01T02:53:58.815 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:58 smithi027 systemd[1]: Stopping Ceph osd.6 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:53:58.815 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:58.585+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:58.815 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:58.585+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:58.815 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:58.585+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:53:59.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:58.813+0000 7ff5ce868700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:53:59.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:58.813+0000 7ff5ce868700 -1 osd.6 130 *** Got signal Terminated *** 2024-06-01T02:53:59.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:58.813+0000 7ff5ce868700 -1 osd.6 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:53:59.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:58.911+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:53:59.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:58.911+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:53:59.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:58 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:58.911+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:53:59.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:59.557+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:53:59.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:59.557+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:53:59.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:53:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:53:59.557+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:00.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:59.959+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:54:00.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:59.959+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:54:00.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:53:59 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:53:59.959+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:54:00.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:00 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:00.602+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:00.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:00 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:00.602+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:00.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:00 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:00.602+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:00.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:00 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:00.602+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:01.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:01.001+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:54:01.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:01.001+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:54:01.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:01.001+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:54:01.967 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:01.615+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:01.967 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:01.615+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:01.967 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:01.615+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:01.967 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:01.615+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:01.967 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:01.966+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:54:01.967 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:01.966+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:54:01.967 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:01.966+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:54:02.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:01 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:01.966+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.785182+0000 front 2024-06-01T02:53:36.785171+0000 (oldest deadline 2024-06-01T02:54:01.485150+0000) 2024-06-01T02:54:02.990 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:02 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:02.623+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:02.990 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:02 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:02.623+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:02.991 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:02 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:02.623+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:02.991 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:02 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:02.623+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:03.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:02 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:02.990+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:14.682011+0000 front 2024-06-01T02:53:14.681961+0000 (oldest deadline 2024-06-01T02:53:38.781780+0000) 2024-06-01T02:54:03.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:02 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:02.990+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.882885+0000 front 2024-06-01T02:53:22.882815+0000 (oldest deadline 2024-06-01T02:53:46.982645+0000) 2024-06-01T02:54:03.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:02 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:02.990+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:30.384011+0000 front 2024-06-01T02:53:30.384095+0000 (oldest deadline 2024-06-01T02:53:52.683953+0000) 2024-06-01T02:54:03.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:02 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6[63117]: 2024-06-01T02:54:02.990+0000 7ff5c6e54700 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.785182+0000 front 2024-06-01T02:53:36.785171+0000 (oldest deadline 2024-06-01T02:54:01.485150+0000) 2024-06-01T02:54:04.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:03 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:03.620+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:04.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:03 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:03.620+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:04.104 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:03 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:03.620+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:04.104 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:03 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:03.620+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:04.573 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:04 smithi027 podman[80382]: 2024-06-01 02:54:04.298137846 +0000 UTC m=+5.590274646 container died 13f0723f71f78d13f0923557c06c27e6d7ff7248f01f0f682f4accea622a50ea (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-06-01T02:54:04.853 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:04 smithi027 podman[80382]: 2024-06-01 02:54:04.747841236 +0000 UTC m=+6.039978035 container cleanup 13f0723f71f78d13f0923557c06c27e6d7ff7248f01f0f682f4accea622a50ea (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-06-01T02:54:04.853 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:04 smithi027 bash[80382]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6 2024-06-01T02:54:04.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:04 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:04.574+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:04.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:04 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:04.574+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:04.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:04 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:04.574+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:04.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:04 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:04.574+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:05.353 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:05 smithi027 podman[80401]: 2024-06-01 02:54:05.058063057 +0000 UTC m=+0.760003165 container remove 13f0723f71f78d13f0923557c06c27e6d7ff7248f01f0f682f4accea622a50ea (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, RELEASE=squid-3d89acc, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux ) 2024-06-01T02:54:05.678 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:05 smithi027 podman[80490]: 2024-06-01 02:54:05.375576581 +0000 UTC m=+0.097163468 container create 392eac6fcf1c674ac17aabb45316b4874bc474b5bc37eb7498dbcc2a345a5c80 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, RELEASE=squid-3d89acc, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528) 2024-06-01T02:54:05.678 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:05 smithi027 podman[80490]: 2024-06-01 02:54:05.296272611 +0000 UTC m=+0.017859492 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:54:05.678 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:05 smithi027 podman[80490]: 2024-06-01 02:54:05.59498217 +0000 UTC m=+0.316569045 container init 392eac6fcf1c674ac17aabb45316b4874bc474b5bc37eb7498dbcc2a345a5c80 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240528, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-06-01T02:54:05.678 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:05 smithi027 podman[80490]: 2024-06-01 02:54:05.598684535 +0000 UTC m=+0.320271413 container start 392eac6fcf1c674ac17aabb45316b4874bc474b5bc37eb7498dbcc2a345a5c80 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240528, ceph=True, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-06-01T02:54:05.678 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:05 smithi027 podman[80490]: 2024-06-01 02:54:05.677836003 +0000 UTC m=+0.399422890 container attach 392eac6fcf1c674ac17aabb45316b4874bc474b5bc37eb7498dbcc2a345a5c80 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240528, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-06-01T02:54:05.678 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:05 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:05.619+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:05.678 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:05 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:05.619+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:05.679 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:05 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:05.619+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:05.679 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:05 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:05.619+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:06.103 INFO:journalctl@ceph.osd.6.smithi027.stdout:Jun 01 02:54:05 smithi027 podman[80490]: 2024-06-01 02:54:05.903324157 +0000 UTC m=+0.624911027 container died 392eac6fcf1c674ac17aabb45316b4874bc474b5bc37eb7498dbcc2a345a5c80 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-3d89acc, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-06-01T02:54:06.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.6.service' 2024-06-01T02:54:06.552 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:54:06.553 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-06-01T02:54:06.553 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-06-01T02:54:06.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.7 2024-06-01T02:54:06.854 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:06.614+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:06.854 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:06.614+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:06.854 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:06.614+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:06.854 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:06.614+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:06.854 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:06 smithi027 systemd[1]: Stopping Ceph osd.7 for e42641dc-1fbc-11ef-bc9b-c7b262605968... 2024-06-01T02:54:07.353 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:06.906+0000 7f7d0db06700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-01T02:54:07.353 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:06.906+0000 7f7d0db06700 -1 osd.7 130 *** Got signal Terminated *** 2024-06-01T02:54:07.353 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:06 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:06.906+0000 7f7d0db06700 -1 osd.7 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-01T02:54:07.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:07 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:07.587+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:07.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:07 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:07.587+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:07.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:07 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:07.587+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:07.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:07 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:07.587+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:08.829 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:08.577+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:08.829 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:08.577+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:08.829 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:08.577+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:08.829 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:08 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:08.577+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:09.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:09 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:09.584+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:09.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:09 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:09.584+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:09.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:09 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:09.584+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:09.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:09 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:09.584+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:10.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:10.572+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:10.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:10.572+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:10.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:10.572+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:10.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:10 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:10.572+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:11.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:11.559+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6806 osd.0 since back 2024-06-01T02:53:11.600786+0000 front 2024-06-01T02:53:11.600812+0000 (oldest deadline 2024-06-01T02:53:35.700835+0000) 2024-06-01T02:54:11.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:11.559+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6816 osd.1 since back 2024-06-01T02:53:22.702469+0000 front 2024-06-01T02:53:22.702510+0000 (oldest deadline 2024-06-01T02:53:47.303164+0000) 2024-06-01T02:54:11.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:11.559+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6824 osd.2 since back 2024-06-01T02:53:27.303744+0000 front 2024-06-01T02:53:27.303756+0000 (oldest deadline 2024-06-01T02:53:51.403650+0000) 2024-06-01T02:54:11.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:11.559+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.3:6830 osd.3 since back 2024-06-01T02:53:36.704757+0000 front 2024-06-01T02:53:36.704690+0000 (oldest deadline 2024-06-01T02:54:00.105032+0000) 2024-06-01T02:54:11.853 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:11 smithi027 ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7[65389]: 2024-06-01T02:54:11.559+0000 7f7d0a0fa700 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6804 osd.4 since back 2024-06-01T02:53:47.106609+0000 front 2024-06-01T02:53:47.106731+0000 (oldest deadline 2024-06-01T02:54:10.606738+0000) 2024-06-01T02:54:12.674 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:12 smithi027 podman[80596]: 2024-06-01 02:54:12.41979313 +0000 UTC m=+5.631125336 container died 196fb45013fd362c862a492923b547fc9d737eb8c78f0946d85df1f7d3c9c039 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-06-01T02:54:13.050 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:12 smithi027 podman[80596]: 2024-06-01 02:54:12.839786421 +0000 UTC m=+6.051118678 container cleanup 196fb45013fd362c862a492923b547fc9d737eb8c78f0946d85df1f7d3c9c039 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-06-01T02:54:13.050 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:12 smithi027 bash[80596]: ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7 2024-06-01T02:54:13.353 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:13 smithi027 podman[80626]: 2024-06-01 02:54:13.049763796 +0000 UTC m=+0.625935933 container remove 196fb45013fd362c862a492923b547fc9d737eb8c78f0946d85df1f7d3c9c039 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-06-01T02:54:13.770 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:13 smithi027 podman[80717]: 2024-06-01 02:54:13.475575264 +0000 UTC m=+0.114958575 container create e4507d0a1a55955cf9f6bbc203a13c558132bf550aa358ce4864c0cc392b2ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-3d89acc, ceph=True, org.label-schema.build-date=20240528, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-06-01T02:54:13.770 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:13 smithi027 podman[80717]: 2024-06-01 02:54:13.378730942 +0000 UTC m=+0.018114253 image pull b045ac545f826a15724cc02d20333d2cc28f7e0eeb5a24d748d09a02280116e1 quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9 2024-06-01T02:54:13.770 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:13 smithi027 podman[80717]: 2024-06-01 02:54:13.687193782 +0000 UTC m=+0.326577110 container init e4507d0a1a55955cf9f6bbc203a13c558132bf550aa358ce4864c0cc392b2ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.schema-version=1.0) 2024-06-01T02:54:13.770 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:13 smithi027 podman[80717]: 2024-06-01 02:54:13.690932735 +0000 UTC m=+0.330316046 container start e4507d0a1a55955cf9f6bbc203a13c558132bf550aa358ce4864c0cc392b2ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-3d89acc, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-06-01T02:54:13.770 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:13 smithi027 podman[80717]: 2024-06-01 02:54:13.769979539 +0000 UTC m=+0.409362850 container attach e4507d0a1a55955cf9f6bbc203a13c558132bf550aa358ce4864c0cc392b2ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, RELEASE=squid-3d89acc, org.label-schema.build-date=20240528) 2024-06-01T02:54:14.103 INFO:journalctl@ceph.osd.7.smithi027.stdout:Jun 01 02:54:13 smithi027 podman[80717]: 2024-06-01 02:54:13.994773561 +0000 UTC m=+0.634156872 container died e4507d0a1a55955cf9f6bbc203a13c558132bf550aa358ce4864c0cc392b2ba9 (image=quay.ceph.io/ceph-ci/ceph@sha256:fe72ccfadf2b7c7c133c6379dcb4e96c4db2f7f512b19bd8eb3efb31f1d8c5d9, name=ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_COMMIT=fe3baff630f29ddc3eb9757f6b798ccb55ab7bd8, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, RELEASE=squid-3d89acc) 2024-06-01T02:54:14.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e42641dc-1fbc-11ef-bc9b-c7b262605968@osd.7.service' 2024-06-01T02:54:14.689 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-01T02:54:14.689 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-06-01T02:54:14.689 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 --force --keep-logs 2024-06-01T02:57:13.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 --force --keep-logs 2024-06-01T02:57:33.679 DEBUG:teuthology.orchestra.run.smithi003:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-01T02:57:33.715 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-01T02:57:33.743 INFO:tasks.cephadm:Archiving crash dumps... 2024-06-01T02:57:33.745 DEBUG:teuthology.misc:Transferring archived files from smithi003:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399/remote/smithi003/crash 2024-06-01T02:57:33.746 DEBUG:teuthology.orchestra.run.smithi003:> sudo tar c -f - -C /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/crash -- . 2024-06-01T02:57:33.801 INFO:teuthology.orchestra.run.smithi003.stderr:tar: /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/crash: Cannot open: No such file or directory 2024-06-01T02:57:33.801 INFO:teuthology.orchestra.run.smithi003.stderr:tar: Error is not recoverable: exiting now 2024-06-01T02:57:33.803 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399/remote/smithi027/crash 2024-06-01T02:57:33.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/crash -- . 2024-06-01T02:57:33.839 INFO:teuthology.orchestra.run.smithi027.stderr:tar: /var/lib/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/crash: Cannot open: No such file or directory 2024-06-01T02:57:33.839 INFO:teuthology.orchestra.run.smithi027.stderr:tar: Error is not recoverable: exiting now 2024-06-01T02:57:33.840 INFO:tasks.cephadm:Checking cluster log for badness... 2024-06-01T02:57:33.840 DEBUG:teuthology.orchestra.run.smithi003:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-01T02:57:33.872 INFO:teuthology.orchestra.run.smithi003.stdout:2024-06-01T02:26:57.925356+0000 mon.smithi003 (mon.0) 742 : cluster [WRN] Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-01T02:57:33.872 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-06-01T02:57:33.872 DEBUG:teuthology.orchestra.run.smithi003:> sudo egrep '\[SEC\]' /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-01T02:57:33.942 DEBUG:teuthology.orchestra.run.smithi003:> sudo egrep '\[ERR\]' /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-01T02:57:34.013 DEBUG:teuthology.orchestra.run.smithi003:> sudo egrep '\[WRN\]' /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-01T02:57:34.082 INFO:teuthology.orchestra.run.smithi003.stdout:2024-06-01T02:26:57.925356+0000 mon.smithi003 (mon.0) 742 : cluster [WRN] Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-01T02:57:34.082 INFO:tasks.cephadm:Compressing logs... 2024-06-01T02:57:34.083 DEBUG:teuthology.orchestra.run.smithi003:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-06-01T02:57:34.125 DEBUG:teuthology.orchestra.run.smithi027:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-06-01T02:57:34.154 INFO:teuthology.orchestra.run.smithi003.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log 2024-06-01T02:57:34.154 INFO:teuthology.orchestra.run.smithi003.stderr:: No such file or directory 2024-06-01T02:57:34.154 INFO:teuthology.orchestra.run.smithi003.stderr:gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.2.log 2024-06-01T02:57:34.154 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mgr.smithi003.njsqxz.log 2024-06-01T02:57:34.155 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.audit.log 2024-06-01T02:57:34.155 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mgr.smithi003.njsqxz.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.3.log 2024-06-01T02:57:34.155 INFO:teuthology.orchestra.run.smithi003.stderr:gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi003.kexxec.log 2024-06-01T02:57:34.155 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.audit.log: /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.1.log 2024-06-01T02:57:34.155 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi003.kexxec.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.0.log 2024-06-01T02:57:34.155 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mon.smithi003.log 2024-06-01T02:57:34.155 INFO:teuthology.orchestra.run.smithi003.stderr: 88.8% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi003.kexxec.log.gz 2024-06-01T02:57:34.155 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.cephadm.log 2024-06-01T02:57:34.157 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mon.smithi003.log: 88.2% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log.gz 2024-06-01T02:57:34.157 INFO:teuthology.orchestra.run.smithi003.stderr: 91.6% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.audit.log.gz 2024-06-01T02:57:34.161 INFO:teuthology.orchestra.run.smithi027.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log 2024-06-01T02:57:34.162 INFO:teuthology.orchestra.run.smithi027.stderr:: No such file or directory 2024-06-01T02:57:34.162 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.audit.log 2024-06-01T02:57:34.162 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.7.log 2024-06-01T02:57:34.162 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.5.log 2024-06-01T02:57:34.162 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi027.xaykap.log 2024-06-01T02:57:34.162 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.4.log 2024-06-01T02:57:34.162 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.5.log: /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi027.xaykap.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mgr.smithi027.tujskj.log 2024-06-01T02:57:34.162 INFO:teuthology.orchestra.run.smithi027.stderr: 87.7% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi027.xaykap.log.gz 2024-06-01T02:57:34.163 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.6.log 2024-06-01T02:57:34.163 INFO:teuthology.orchestra.run.smithi027.stderr:gzip/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mgr.smithi027.tujskj.log: -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mon.smithi027.log 2024-06-01T02:57:34.163 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.cephadm.log 2024-06-01T02:57:34.163 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mon.smithi027.log: 88.3% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.log.gz 2024-06-01T02:57:34.163 INFO:teuthology.orchestra.run.smithi027.stderr: 91.7% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.audit.log.gz 2024-06-01T02:57:34.166 INFO:teuthology.orchestra.run.smithi027.stderr:gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-volume.log 2024-06-01T02:57:34.166 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-06-01T02:57:34.166 INFO:teuthology.orchestra.run.smithi027.stderr: 85.8% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.cephadm.log.gz/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-volume.log: 2024-06-01T02:57:34.167 INFO:teuthology.orchestra.run.smithi003.stderr:gzip -5 --verbose -- /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-volume.log 2024-06-01T02:57:34.167 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-06-01T02:57:34.167 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-volume.log: 85.9% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph.cephadm.log.gz 2024-06-01T02:57:34.181 INFO:teuthology.orchestra.run.smithi027.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-volume.log.gz 2024-06-01T02:57:34.184 INFO:teuthology.orchestra.run.smithi003.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-volume.log.gz 2024-06-01T02:57:34.186 INFO:teuthology.orchestra.run.smithi027.stderr: 90.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-06-01T02:57:34.199 INFO:teuthology.orchestra.run.smithi003.stderr: 91.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-06-01T02:57:34.205 INFO:teuthology.orchestra.run.smithi027.stderr: 89.9% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mgr.smithi027.tujskj.log.gz 2024-06-01T02:57:34.521 INFO:teuthology.orchestra.run.smithi003.stderr: 89.3% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mgr.smithi003.njsqxz.log.gz 2024-06-01T02:57:34.733 INFO:teuthology.orchestra.run.smithi027.stderr: 92.6% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mon.smithi027.log.gz 2024-06-01T02:57:35.763 INFO:teuthology.orchestra.run.smithi003.stderr: 91.3% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-mon.smithi003.log.gz 2024-06-01T02:57:42.589 INFO:teuthology.orchestra.run.smithi003.stderr: 94.4% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.2.log.gz 2024-06-01T02:57:44.786 INFO:teuthology.orchestra.run.smithi027.stderr: 93.9% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.4.log.gz 2024-06-01T02:57:46.001 INFO:teuthology.orchestra.run.smithi027.stderr: 94.2% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.6.log.gz 2024-06-01T02:57:46.717 INFO:teuthology.orchestra.run.smithi003.stderr: 94.1% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.0.log.gz 2024-06-01T02:57:47.456 INFO:teuthology.orchestra.run.smithi003.stderr: 94.2% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.1.log.gz 2024-06-01T02:57:48.063 INFO:teuthology.orchestra.run.smithi003.stderr: 94.0% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.3.log.gz 2024-06-01T02:57:48.066 INFO:teuthology.orchestra.run.smithi003.stderr: 2024-06-01T02:57:48.066 INFO:teuthology.orchestra.run.smithi003.stderr:real 0m13.927s 2024-06-01T02:57:48.066 INFO:teuthology.orchestra.run.smithi003.stderr:user 0m48.479s 2024-06-01T02:57:48.066 INFO:teuthology.orchestra.run.smithi003.stderr:sys 0m1.716s 2024-06-01T02:57:48.682 INFO:teuthology.orchestra.run.smithi027.stderr: 94.1% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.5.log.gz 2024-06-01T02:57:53.126 INFO:teuthology.orchestra.run.smithi027.stderr: 94.1% -- replaced with /var/log/ceph/e42641dc-1fbc-11ef-bc9b-c7b262605968/ceph-osd.7.log.gz 2024-06-01T02:57:53.128 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-06-01T02:57:53.128 INFO:teuthology.orchestra.run.smithi027.stderr:real 0m18.987s 2024-06-01T02:57:53.129 INFO:teuthology.orchestra.run.smithi027.stderr:user 0m54.595s 2024-06-01T02:57:53.129 INFO:teuthology.orchestra.run.smithi027.stderr:sys 0m1.975s 2024-06-01T02:57:53.129 INFO:tasks.cephadm:Archiving logs... 2024-06-01T02:57:53.129 DEBUG:teuthology.misc:Transferring archived files from smithi003:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399/remote/smithi003/log 2024-06-01T02:57:53.130 DEBUG:teuthology.orchestra.run.smithi003:> sudo tar c -f - -C /var/log/ceph -- . 2024-06-01T02:57:59.565 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399/remote/smithi027/log 2024-06-01T02:57:59.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/log/ceph -- . 2024-06-01T02:58:07.003 INFO:tasks.cephadm:Removing cluster... 2024-06-01T02:58:07.003 DEBUG:teuthology.orchestra.run.smithi003:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 --force 2024-06-01T02:58:07.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e42641dc-1fbc-11ef-bc9b-c7b262605968 --force 2024-06-01T02:58:07.736 INFO:tasks.cephadm:Removing cephadm ... 2024-06-01T02:58:07.736 DEBUG:teuthology.orchestra.run.smithi003:> rm -rf /home/ubuntu/cephtest/cephadm 2024-06-01T02:58:07.753 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf /home/ubuntu/cephtest/cephadm 2024-06-01T02:58:07.768 INFO:tasks.cephadm:Teardown complete 2024-06-01T02:58:07.769 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-06-01T02:58:07.779 INFO:teuthology.task.clock:Checking final clock skew... 2024-06-01T02:58:07.779 DEBUG:teuthology.orchestra.run.smithi003:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-01T02:58:07.796 DEBUG:teuthology.orchestra.run.smithi027:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-01T02:58:07.809 INFO:teuthology.orchestra.run.smithi003.stderr:bash: line 1: ntpq: command not found 2024-06-01T02:58:07.814 INFO:teuthology.orchestra.run.smithi003.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-01T02:58:07.814 INFO:teuthology.orchestra.run.smithi003.stdout:=============================================================================== 2024-06-01T02:58:07.814 INFO:teuthology.orchestra.run.smithi003.stdout:^* hv01.front.sepia.ceph.com 3 6 377 62 +827us[ +794us] +/- 47ms 2024-06-01T02:58:07.814 INFO:teuthology.orchestra.run.smithi003.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 3 -1299us[-1299us] +/- 63ms 2024-06-01T02:58:07.814 INFO:teuthology.orchestra.run.smithi003.stdout:^+ hv03.front.sepia.ceph.com 3 8 377 129 +292us[ +250us] +/- 68ms 2024-06-01T02:58:07.814 INFO:teuthology.orchestra.run.smithi003.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:58:07.822 INFO:teuthology.orchestra.run.smithi027.stderr:bash: line 1: ntpq: command not found 2024-06-01T02:58:07.826 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-01T02:58:07.826 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2024-06-01T02:58:07.826 INFO:teuthology.orchestra.run.smithi027.stdout:^* hv01.front.sepia.ceph.com 3 6 377 62 +863us[ +855us] +/- 47ms 2024-06-01T02:58:07.826 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 62 -1266us[-1274us] +/- 62ms 2024-06-01T02:58:07.826 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 58 +277us[ +277us] +/- 69ms 2024-06-01T02:58:07.826 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-06-01T02:58:07.827 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-06-01T02:58:07.836 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-06-01T02:58:07.837 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-06-01T02:58:07.847 DEBUG:teuthology.orchestra.run.smithi003:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-06-01T02:58:07.940 DEBUG:teuthology.orchestra.run.smithi027:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-06-01T02:58:08.016 DEBUG:teuthology.orchestra.run.smithi003:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-06-01T02:58:08.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-01T02:58:08.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-06-01T02:58:08.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-01T02:58:08.077 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-06-01T02:58:08.089 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-06-01T02:58:08.104 INFO:teuthology.task.internal:Duration was 2798.472007 seconds 2024-06-01T02:58:08.105 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-06-01T02:58:08.118 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-06-01T02:58:08.118 DEBUG:teuthology.orchestra.run.smithi003:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-06-01T02:58:08.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-06-01T02:58:08.159 INFO:teuthology.orchestra.run.smithi003.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-01T02:58:08.160 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-01T02:58:08.630 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-06-01T02:58:08.630 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi003.front.sepia.ceph.com 2024-06-01T02:58:08.630 DEBUG:teuthology.orchestra.run.smithi003:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-06-01T02:58:08.692 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi027.front.sepia.ceph.com 2024-06-01T02:58:08.692 DEBUG:teuthology.orchestra.run.smithi027:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-06-01T02:58:08.726 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-06-01T02:58:08.726 DEBUG:teuthology.orchestra.run.smithi003:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-06-01T02:58:08.734 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-06-01T02:58:08.794 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-06-01T02:58:08.795 DEBUG:teuthology.orchestra.run.smithi003:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-06-01T02:58:08.802 DEBUG:teuthology.orchestra.run.smithi027:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-06-01T02:58:21.322 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-06-01T02:58:21.333 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-06-01T02:58:21.333 DEBUG:teuthology.orchestra.run.smithi003:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-06-01T02:58:21.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-06-01T02:58:21.390 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-06-01T02:58:21.402 DEBUG:teuthology.orchestra.run.smithi003:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-06-01T02:58:21.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-06-01T02:58:21.425 INFO:teuthology.orchestra.run.smithi003.stdout:kernel.core_pattern = core 2024-06-01T02:58:21.458 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = core 2024-06-01T02:58:21.489 DEBUG:teuthology.orchestra.run.smithi003:> test -e /home/ubuntu/cephtest/archive/coredump 2024-06-01T02:58:21.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-01T02:58:21.525 DEBUG:teuthology.orchestra.run.smithi027:> test -e /home/ubuntu/cephtest/archive/coredump 2024-06-01T02:58:21.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-01T02:58:21.546 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-06-01T02:58:21.556 INFO:teuthology.task.internal:Transferring archived files... 2024-06-01T02:58:21.557 DEBUG:teuthology.misc:Transferring archived files from smithi003:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399/remote/smithi003 2024-06-01T02:58:21.557 DEBUG:teuthology.orchestra.run.smithi003:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-06-01T02:58:21.627 DEBUG:teuthology.misc:Transferring archived files from smithi027:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-05-31_21:08:02-orch-squid-distro-default-smithi/7736399/remote/smithi027 2024-06-01T02:58:21.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-06-01T02:58:21.669 INFO:teuthology.task.internal:Removing archive directory... 2024-06-01T02:58:21.669 DEBUG:teuthology.orchestra.run.smithi003:> rm -rf -- /home/ubuntu/cephtest/archive 2024-06-01T02:58:21.672 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf -- /home/ubuntu/cephtest/archive 2024-06-01T02:58:21.720 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-06-01T02:58:21.732 INFO:teuthology.task.internal:Not uploading archives. 2024-06-01T02:58:21.732 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-06-01T02:58:21.743 INFO:teuthology.task.internal:Tidying up after the test... 2024-06-01T02:58:21.743 DEBUG:teuthology.orchestra.run.smithi003:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-06-01T02:58:21.745 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-06-01T02:58:21.758 INFO:teuthology.orchestra.run.smithi003.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 1 02:58 /home/ubuntu/cephtest 2024-06-01T02:58:21.776 INFO:teuthology.orchestra.run.smithi027.stdout: 265916 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 1 02:58 /home/ubuntu/cephtest 2024-06-01T02:58:21.778 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-06-01T02:58:21.788 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-06-01T02:58:21.822 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2798.4720072746277 failure_reason: '"2024-06-01T02:26:57.925356+0000 mon.smithi003 (mon.0) 742 : cluster [WRN] Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-06-01T02:58:21.823 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-06-01T02:58:21.895 INFO:teuthology.run:FAIL