2024-09-13T22:13:18.568 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T22:13:18.592 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T22:13:18.686 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904293 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7904293' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 3819 sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 sleep_before_teardown: 0 subset: 43/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 targets: smithi105.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEcVqvaXcQmECAZ+RHrfmvN/Gfrqkxhc4uPaNUGWmbvyJvM2QMDm2Sm8tKaf1T4KnOrLwC5xs586pV6ycyCUcLs= smithi118.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP2XZNJJHGcCDfr0TPzS3bWVYcZmxoVt5qbxDRsjMWOj/BTAkJuPIKk7Nz/Z35vpEGgXFrFmzXqfAwIm6k+5yMU= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_21:08:03 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-13T22:13:18.687 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa; will attempt to use it 2024-09-13T22:13:18.687 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_ecab7be58963a5bd03910cd73eef66405d7fdd50/qa/tasks 2024-09-13T22:13:18.687 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T22:13:18.689 INFO:teuthology.task.internal:Checking packages... 2024-09-13T22:13:18.709 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'ecab7be58963a5bd03910cd73eef66405d7fdd50' 2024-09-13T22:13:18.709 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T22:13:18.710 INFO:teuthology.packaging:ref: None 2024-09-13T22:13:18.710 INFO:teuthology.packaging:tag: None 2024-09-13T22:13:18.710 INFO:teuthology.packaging:branch: squid 2024-09-13T22:13:18.710 INFO:teuthology.packaging:sha1: ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:13:18.710 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-13T22:13:18.950 INFO:teuthology.task.internal:Found packages for ceph version 19.1.1-324.gecab7be5 2024-09-13T22:13:18.951 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T22:13:18.960 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T22:13:18.960 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T22:13:18.972 INFO:teuthology.task.internal:Saving configuration 2024-09-13T22:13:18.986 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T22:13:18.994 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T22:13:19.028 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi105.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904293', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 22:07:41.593028', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEcVqvaXcQmECAZ+RHrfmvN/Gfrqkxhc4uPaNUGWmbvyJvM2QMDm2Sm8tKaf1T4KnOrLwC5xs586pV6ycyCUcLs='} 2024-09-13T22:13:19.059 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi118.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904293', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 22:07:41.594018', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP2XZNJJHGcCDfr0TPzS3bWVYcZmxoVt5qbxDRsjMWOj/BTAkJuPIKk7Nz/Z35vpEGgXFrFmzXqfAwIm6k+5yMU='} 2024-09-13T22:13:19.059 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T22:13:19.066 INFO:teuthology.task.internal:roles: ubuntu@smithi105.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-13T22:13:19.066 INFO:teuthology.task.internal:roles: ubuntu@smithi118.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-13T22:13:19.066 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T22:13:19.145 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fce3b197ac0>, signals=[15]) 2024-09-13T22:13:19.146 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T22:13:19.153 INFO:teuthology.task.internal:Opening connections... 2024-09-13T22:13:19.153 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:13:19.155 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:13:19.257 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi118.front.sepia.ceph.com 2024-09-13T22:13:19.258 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:13:19.339 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T22:13:19.345 DEBUG:teuthology.orchestra.run.smithi105:> uname -m 2024-09-13T22:13:19.364 INFO:teuthology.orchestra.run.smithi105.stdout:x86_64 2024-09-13T22:13:19.365 DEBUG:teuthology.orchestra.run.smithi105:> cat /etc/os-release 2024-09-13T22:13:19.422 INFO:teuthology.orchestra.run.smithi105.stdout:NAME="CentOS Stream" 2024-09-13T22:13:19.422 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION="9" 2024-09-13T22:13:19.422 INFO:teuthology.orchestra.run.smithi105.stdout:ID="centos" 2024-09-13T22:13:19.422 INFO:teuthology.orchestra.run.smithi105.stdout:ID_LIKE="rhel fedora" 2024-09-13T22:13:19.422 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION_ID="9" 2024-09-13T22:13:19.422 INFO:teuthology.orchestra.run.smithi105.stdout:PLATFORM_ID="platform:el9" 2024-09-13T22:13:19.423 INFO:teuthology.orchestra.run.smithi105.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T22:13:19.423 INFO:teuthology.orchestra.run.smithi105.stdout:ANSI_COLOR="0;31" 2024-09-13T22:13:19.423 INFO:teuthology.orchestra.run.smithi105.stdout:LOGO="fedora-logo-icon" 2024-09-13T22:13:19.423 INFO:teuthology.orchestra.run.smithi105.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T22:13:19.423 INFO:teuthology.orchestra.run.smithi105.stdout:HOME_URL="https://centos.org/" 2024-09-13T22:13:19.423 INFO:teuthology.orchestra.run.smithi105.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T22:13:19.423 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T22:13:19.423 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T22:13:19.424 INFO:teuthology.lock.ops:Updating smithi105.front.sepia.ceph.com on lock server 2024-09-13T22:13:19.454 DEBUG:teuthology.orchestra.run.smithi118:> uname -m 2024-09-13T22:13:19.472 INFO:teuthology.orchestra.run.smithi118.stdout:x86_64 2024-09-13T22:13:19.473 DEBUG:teuthology.orchestra.run.smithi118:> cat /etc/os-release 2024-09-13T22:13:19.530 INFO:teuthology.orchestra.run.smithi118.stdout:NAME="CentOS Stream" 2024-09-13T22:13:19.530 INFO:teuthology.orchestra.run.smithi118.stdout:VERSION="9" 2024-09-13T22:13:19.530 INFO:teuthology.orchestra.run.smithi118.stdout:ID="centos" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:ID_LIKE="rhel fedora" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:VERSION_ID="9" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:PLATFORM_ID="platform:el9" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:ANSI_COLOR="0;31" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:LOGO="fedora-logo-icon" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:HOME_URL="https://centos.org/" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T22:13:19.531 INFO:teuthology.orchestra.run.smithi118.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T22:13:19.532 INFO:teuthology.lock.ops:Updating smithi118.front.sepia.ceph.com on lock server 2024-09-13T22:13:19.562 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T22:13:19.572 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T22:13:19.583 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T22:13:19.584 DEBUG:teuthology.orchestra.run.smithi105:> test '!' -e /home/ubuntu/cephtest 2024-09-13T22:13:19.586 DEBUG:teuthology.orchestra.run.smithi118:> test '!' -e /home/ubuntu/cephtest 2024-09-13T22:13:19.603 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T22:13:19.610 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T22:13:19.610 DEBUG:teuthology.orchestra.run.smithi105:> test -z $(ls -A /var/lib/ceph) 2024-09-13T22:13:19.644 DEBUG:teuthology.orchestra.run.smithi118:> test -z $(ls -A /var/lib/ceph) 2024-09-13T22:13:19.751 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T22:13:19.819 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T22:13:19.833 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T22:13:19.833 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T22:13:19.834 DEBUG:teuthology.orchestra.run.smithi105:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T22:13:19.834 DEBUG:teuthology.orchestra.run.smithi118:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T22:13:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:13:19.852 DEBUG:teuthology.orchestra.run.smithi118:> uname -r 2024-09-13T22:13:19.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:13:19.853 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-13T22:13:19.908 INFO:teuthology.orchestra.run.smithi118.stdout:5.14.0-503.el9.x86_64 2024-09-13T22:13:19.909 INFO:teuthology.task.kernel:Running kernel on smithi118: 5.14.0-503.el9.x86_64 2024-09-13T22:13:19.909 DEBUG:teuthology.orchestra.run.smithi118:> sudo yum install -y kernel 2024-09-13T22:13:19.910 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-503.el9.x86_64 2024-09-13T22:13:19.911 INFO:teuthology.task.kernel:Running kernel on smithi105: 5.14.0-503.el9.x86_64 2024-09-13T22:13:19.911 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum install -y kernel 2024-09-13T22:13:22.509 INFO:teuthology.orchestra.run.smithi118.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.2 MB 00:00 2024-09-13T22:13:23.280 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - BaseOS 4.8 MB/s | 8.2 MB 00:01 2024-09-13T22:13:25.248 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - AppStream 24 MB/s | 20 MB 00:00 2024-09-13T22:13:25.785 INFO:teuthology.orchestra.run.smithi118.stdout:CentOS Stream 9 - AppStream 9.4 MB/s | 20 MB 00:02 2024-09-13T22:13:29.673 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - CRB 9.8 MB/s | 6.5 MB 00:00 2024-09-13T22:13:31.477 INFO:teuthology.orchestra.run.smithi118.stdout:CentOS Stream 9 - CRB 3.5 MB/s | 6.5 MB 00:01 2024-09-13T22:13:32.099 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 19 kB 00:01 2024-09-13T22:13:33.362 INFO:teuthology.orchestra.run.smithi105.stdout:Extra Packages for Enterprise Linux 20 MB/s | 23 MB 00:01 2024-09-13T22:13:33.785 INFO:teuthology.orchestra.run.smithi118.stdout:CentOS Stream 9 - Extras packages 17 kB/s | 19 kB 00:01 2024-09-13T22:13:34.681 INFO:teuthology.orchestra.run.smithi118.stdout:Extra Packages for Enterprise Linux 30 MB/s | 23 MB 00:00 2024-09-13T22:13:38.784 INFO:teuthology.orchestra.run.smithi105.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-13T22:13:40.121 INFO:teuthology.orchestra.run.smithi118.stdout:lab-extras 25 kB/s | 1.7 kB 00:00 2024-09-13T22:13:40.782 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T22:13:40.782 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T22:13:40.895 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout: Package Architecture Version Repository Size 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-13T22:13:40.902 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-13T22:13:40.903 INFO:teuthology.orchestra.run.smithi105.stdout:Install 4 Packages 2024-09-13T22:13:40.903 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:13:40.903 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 79 M 2024-09-13T22:13:40.903 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 126 M 2024-09-13T22:13:40.903 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-13T22:13:41.485 INFO:teuthology.orchestra.run.smithi105.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 41 kB/s | 23 kB 00:00 2024-09-13T22:13:42.110 INFO:teuthology.orchestra.run.smithi118.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T22:13:42.110 INFO:teuthology.orchestra.run.smithi118.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T22:13:42.208 INFO:teuthology.orchestra.run.smithi118.stdout:Dependencies resolved. 2024-09-13T22:13:42.219 INFO:teuthology.orchestra.run.smithi118.stdout:================================================================================ 2024-09-13T22:13:42.219 INFO:teuthology.orchestra.run.smithi118.stdout: Package Architecture Version Repository Size 2024-09-13T22:13:42.219 INFO:teuthology.orchestra.run.smithi118.stdout:================================================================================ 2024-09-13T22:13:42.219 INFO:teuthology.orchestra.run.smithi118.stdout:Installing: 2024-09-13T22:13:42.219 INFO:teuthology.orchestra.run.smithi118.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T22:13:42.219 INFO:teuthology.orchestra.run.smithi118.stdout:Installing dependencies: 2024-09-13T22:13:42.220 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T22:13:42.220 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T22:13:42.220 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T22:13:42.220 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:13:42.220 INFO:teuthology.orchestra.run.smithi118.stdout:Transaction Summary 2024-09-13T22:13:42.220 INFO:teuthology.orchestra.run.smithi118.stdout:================================================================================ 2024-09-13T22:13:42.220 INFO:teuthology.orchestra.run.smithi118.stdout:Install 4 Packages 2024-09-13T22:13:42.220 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:13:42.221 INFO:teuthology.orchestra.run.smithi118.stdout:Total download size: 79 M 2024-09-13T22:13:42.221 INFO:teuthology.orchestra.run.smithi118.stdout:Installed size: 126 M 2024-09-13T22:13:42.221 INFO:teuthology.orchestra.run.smithi118.stdout:Downloading Packages: 2024-09-13T22:13:42.344 INFO:teuthology.orchestra.run.smithi105.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-09-13T22:13:42.572 INFO:teuthology.orchestra.run.smithi118.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 77 kB/s | 23 kB 00:00 2024-09-13T22:13:42.961 INFO:teuthology.orchestra.run.smithi105.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 17 MB/s | 35 MB 00:02 2024-09-13T22:13:44.494 INFO:teuthology.orchestra.run.smithi105.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 9.5 MB/s | 29 MB 00:03 2024-09-13T22:13:44.494 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-13T22:13:44.495 INFO:teuthology.orchestra.run.smithi105.stdout:Total 22 MB/s | 79 MB 00:03 2024-09-13T22:13:45.185 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-13T22:13:45.550 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-13T22:13:45.550 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-13T22:13:45.722 INFO:teuthology.orchestra.run.smithi118.stdout:(2/4): kernel-modules-core-5.14.0-505.el9.x86_6 9.1 MB/s | 29 MB 00:03 2024-09-13T22:13:46.488 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-13T22:13:46.489 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-13T22:13:48.013 INFO:teuthology.orchestra.run.smithi118.stdout:(3/4): kernel-core-5.14.0-505.el9.x86_64.rpm 2.7 MB/s | 16 MB 00:05 2024-09-13T22:13:48.941 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-13T22:13:49.627 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T22:13:49.719 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:13:49.821 INFO:teuthology.orchestra.run.smithi118.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 4.6 MB/s | 35 MB 00:07 2024-09-13T22:13:49.822 INFO:teuthology.orchestra.run.smithi118.stdout:-------------------------------------------------------------------------------- 2024-09-13T22:13:49.822 INFO:teuthology.orchestra.run.smithi118.stdout:Total 10 MB/s | 79 MB 00:07 2024-09-13T22:13:50.506 INFO:teuthology.orchestra.run.smithi118.stdout:Running transaction check 2024-09-13T22:13:50.851 INFO:teuthology.orchestra.run.smithi118.stdout:Transaction check succeeded. 2024-09-13T22:13:50.851 INFO:teuthology.orchestra.run.smithi118.stdout:Running transaction test 2024-09-13T22:13:51.818 INFO:teuthology.orchestra.run.smithi118.stdout:Transaction test succeeded. 2024-09-13T22:13:51.818 INFO:teuthology.orchestra.run.smithi118.stdout:Running transaction 2024-09-13T22:13:52.518 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:13:52.657 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:13:54.295 INFO:teuthology.orchestra.run.smithi118.stdout: Preparing : 1/1 2024-09-13T22:13:54.996 INFO:teuthology.orchestra.run.smithi118.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T22:13:55.071 INFO:teuthology.orchestra.run.smithi118.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:13:57.484 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:13:57.515 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:13:57.881 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:13:58.010 INFO:teuthology.orchestra.run.smithi118.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:14:02.376 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:02.845 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:14:02.961 INFO:teuthology.orchestra.run.smithi118.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:07.804 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:40.832 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:40.891 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:41.243 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:41.243 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T22:14:41.243 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:14:41.243 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:14:42.054 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:42.054 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:14:42.054 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-13T22:14:42.054 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T22:14:42.054 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T22:14:42.054 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T22:14:42.054 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T22:14:42.054 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:14:42.054 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-13T22:14:42.434 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T22:14:42.515 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T22:14:42.516 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T22:14:42.516 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T22:14:42.516 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T22:14:42.516 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T22:14:42.516 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-13T22:14:42.516 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi105.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T22:14:42.517 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T22:14:42.575 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T22:14:42.575 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T22:14:42.575 DEBUG:teuthology.orchestra.run.smithi105:> sudo rpm -qi grub2-tools 2024-09-13T22:14:42.613 INFO:teuthology.orchestra.run.smithi105.stdout:Name : grub2-tools 2024-09-13T22:14:42.613 INFO:teuthology.orchestra.run.smithi105.stdout:Epoch : 1 2024-09-13T22:14:42.613 INFO:teuthology.orchestra.run.smithi105.stdout:Version : 2.06 2024-09-13T22:14:42.613 INFO:teuthology.orchestra.run.smithi105.stdout:Release : 82.el9 2024-09-13T22:14:42.613 INFO:teuthology.orchestra.run.smithi105.stdout:Architecture: x86_64 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Group : Unspecified 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Size : 8274670 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:License : GPLv3+ 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Packager : builder@centos.org 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Vendor : CentOS 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Summary : Support tools for GRUB. 2024-09-13T22:14:42.614 INFO:teuthology.orchestra.run.smithi105.stdout:Description : 2024-09-13T22:14:42.615 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:14:42.615 INFO:teuthology.orchestra.run.smithi105.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T22:14:42.615 INFO:teuthology.orchestra.run.smithi105.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T22:14:42.615 INFO:teuthology.orchestra.run.smithi105.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T22:14:42.615 INFO:teuthology.orchestra.run.smithi105.stdout:hardware devices. 2024-09-13T22:14:42.615 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:14:42.615 INFO:teuthology.orchestra.run.smithi105.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T22:14:42.616 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T22:14:42.616 INFO:teuthology.task.kernel:Updating grub on smithi105 to boot 5.14.0-505.el9.x86_64 2024-09-13T22:14:42.616 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T22:14:43.209 INFO:teuthology.orchestra.run.smithi105.stderr:Generating grub configuration file ... 2024-09-13T22:14:44.575 INFO:teuthology.orchestra.run.smithi105.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T22:14:44.608 INFO:teuthology.orchestra.run.smithi105.stderr:done 2024-09-13T22:14:44.610 DEBUG:teuthology.orchestra.run.smithi105:> mktemp 2024-09-13T22:14:44.626 INFO:teuthology.orchestra.run.smithi105.stdout:/tmp/tmp.wfvSXMsFee 2024-09-13T22:14:44.626 DEBUG:teuthology.orchestra.run.smithi105:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.wfvSXMsFee 2024-09-13T22:14:44.700 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0666 /tmp/tmp.wfvSXMsFee 2024-09-13T22:14:44.837 DEBUG:teuthology.orchestra.remote:smithi105:/tmp/tmp.wfvSXMsFee is 6KB 2024-09-13T22:14:44.889 DEBUG:teuthology.orchestra.run.smithi105:> rm -fr /tmp/tmp.wfvSXMsFee 2024-09-13T22:14:44.904 DEBUG:teuthology.orchestra.run.smithi105:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T22:14:44.966 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T22:14:44.966 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T22:14:44.966 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T22:14:44.967 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T22:14:44.968 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T22:14:45.125 DEBUG:teuthology.orchestra.run.smithi105:> sudo shutdown -r now 2024-09-13T22:14:46.546 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:46.603 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:46.994 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:46.994 INFO:teuthology.orchestra.run.smithi118.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T22:14:46.995 INFO:teuthology.orchestra.run.smithi118.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T22:14:46.995 INFO:teuthology.orchestra.run.smithi118.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T22:14:47.892 INFO:teuthology.orchestra.run.smithi118.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T22:14:47.892 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:14:47.892 INFO:teuthology.orchestra.run.smithi118.stdout:Installed: 2024-09-13T22:14:47.892 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T22:14:47.892 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T22:14:47.893 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T22:14:47.893 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T22:14:47.893 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:14:47.893 INFO:teuthology.orchestra.run.smithi118.stdout:Complete! 2024-09-13T22:14:48.313 DEBUG:teuthology.orchestra.run.smithi118:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T22:14:48.398 INFO:teuthology.orchestra.run.smithi118.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T22:14:48.398 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T22:14:48.398 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T22:14:48.398 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T22:14:48.399 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T22:14:48.399 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-13T22:14:48.399 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi118.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T22:14:48.399 DEBUG:teuthology.orchestra.run.smithi118:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T22:14:48.458 INFO:teuthology.orchestra.run.smithi118.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T22:14:48.458 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T22:14:48.458 DEBUG:teuthology.orchestra.run.smithi118:> sudo rpm -qi grub2-tools 2024-09-13T22:14:48.490 INFO:teuthology.orchestra.run.smithi118.stdout:Name : grub2-tools 2024-09-13T22:14:48.490 INFO:teuthology.orchestra.run.smithi118.stdout:Epoch : 1 2024-09-13T22:14:48.490 INFO:teuthology.orchestra.run.smithi118.stdout:Version : 2.06 2024-09-13T22:14:48.490 INFO:teuthology.orchestra.run.smithi118.stdout:Release : 82.el9 2024-09-13T22:14:48.490 INFO:teuthology.orchestra.run.smithi118.stdout:Architecture: x86_64 2024-09-13T22:14:48.490 INFO:teuthology.orchestra.run.smithi118.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T22:14:48.490 INFO:teuthology.orchestra.run.smithi118.stdout:Group : Unspecified 2024-09-13T22:14:48.490 INFO:teuthology.orchestra.run.smithi118.stdout:Size : 8274670 2024-09-13T22:14:48.490 INFO:teuthology.orchestra.run.smithi118.stdout:License : GPLv3+ 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:Packager : builder@centos.org 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:Vendor : CentOS 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:Summary : Support tools for GRUB. 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:Description : 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout:hardware devices. 2024-09-13T22:14:48.491 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:14:48.492 INFO:teuthology.orchestra.run.smithi118.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T22:14:48.492 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T22:14:48.492 INFO:teuthology.task.kernel:Updating grub on smithi118 to boot 5.14.0-505.el9.x86_64 2024-09-13T22:14:48.493 DEBUG:teuthology.orchestra.run.smithi118:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T22:14:49.103 INFO:teuthology.orchestra.run.smithi118.stderr:Generating grub configuration file ... 2024-09-13T22:14:50.452 INFO:teuthology.orchestra.run.smithi118.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T22:14:50.486 INFO:teuthology.orchestra.run.smithi118.stderr:done 2024-09-13T22:14:50.486 DEBUG:teuthology.orchestra.run.smithi118:> mktemp 2024-09-13T22:14:50.501 INFO:teuthology.orchestra.run.smithi118.stdout:/tmp/tmp.1vhN53iEZl 2024-09-13T22:14:50.502 DEBUG:teuthology.orchestra.run.smithi118:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.1vhN53iEZl 2024-09-13T22:14:50.567 DEBUG:teuthology.orchestra.run.smithi118:> sudo chmod 0666 /tmp/tmp.1vhN53iEZl 2024-09-13T22:14:50.713 DEBUG:teuthology.orchestra.remote:smithi118:/tmp/tmp.1vhN53iEZl is 6KB 2024-09-13T22:14:50.734 DEBUG:teuthology.orchestra.run.smithi118:> rm -fr /tmp/tmp.1vhN53iEZl 2024-09-13T22:14:50.748 DEBUG:teuthology.orchestra.run.smithi118:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T22:14:50.810 INFO:teuthology.orchestra.run.smithi118.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T22:14:50.811 INFO:teuthology.orchestra.run.smithi118.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T22:14:50.811 INFO:teuthology.orchestra.run.smithi118.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T22:14:50.811 INFO:teuthology.orchestra.run.smithi118.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T22:14:50.812 DEBUG:teuthology.orchestra.run.smithi118:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T22:14:50.964 DEBUG:teuthology.orchestra.run.smithi118:> sudo shutdown -r now 2024-09-13T22:15:15.149 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-13T22:15:15.149 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-13T22:15:15.150 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:15:20.973 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-13T22:15:20.973 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-13T22:15:20.974 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:15:33.693 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.105 2024-09-13T22:15:39.325 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.118 2024-09-13T22:15:42.697 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-13T22:15:42.698 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:15:45.761 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.105 2024-09-13T22:15:48.329 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-13T22:15:48.329 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:15:51.425 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.118 2024-09-13T22:15:57.767 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-13T22:15:57.768 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:16:03.429 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-13T22:16:03.430 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:16:29.594 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-13T22:16:30.018 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-13T22:16:30.018 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T22:16:30.019 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-13T22:16:30.033 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-505.el9.x86_64 2024-09-13T22:16:30.033 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T22:16:30.033 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T22:16:30.033 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T22:16:31.034 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-13T22:16:31.034 DEBUG:teuthology.orchestra.run.smithi105:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T22:16:31.091 INFO:teuthology.orchestra.run.smithi105.stdout:ttyS1 2024-09-13T22:16:31.114 DEBUG:teuthology.parallel:result is None 2024-09-13T22:17:03.441 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T22:17:18.442 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-13T22:17:18.443 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:17:18.940 DEBUG:teuthology.orchestra.run.smithi118:> true 2024-09-13T22:17:19.290 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-13T22:17:19.291 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T22:17:19.291 DEBUG:teuthology.orchestra.run.smithi118:> uname -r 2024-09-13T22:17:19.309 INFO:teuthology.orchestra.run.smithi118.stdout:5.14.0-505.el9.x86_64 2024-09-13T22:17:19.309 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T22:17:19.309 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T22:17:19.309 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T22:17:20.310 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-13T22:17:20.311 DEBUG:teuthology.orchestra.run.smithi118:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T22:17:20.397 INFO:teuthology.orchestra.run.smithi118.stdout:ttyS1 2024-09-13T22:17:20.418 DEBUG:teuthology.parallel:result is None 2024-09-13T22:17:20.418 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T22:17:20.426 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T22:17:20.426 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T22:17:20.429 DEBUG:teuthology.orchestra.run.smithi118:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T22:17:20.447 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T22:17:20.454 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T22:17:20.463 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T22:17:20.463 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T22:17:20.487 DEBUG:teuthology.orchestra.run.smithi118:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T22:17:20.519 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T22:17:20.525 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T22:17:20.525 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T22:17:20.557 DEBUG:teuthology.orchestra.run.smithi118:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T22:17:20.594 INFO:teuthology.orchestra.run.smithi118.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T22:17:20.603 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T22:17:20.605 INFO:teuthology.orchestra.run.smithi118.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T22:17:20.615 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T22:17:20.616 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T22:17:20.623 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T22:17:20.624 DEBUG:teuthology.orchestra.run.smithi105:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T22:17:20.626 DEBUG:teuthology.orchestra.run.smithi118:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T22:17:20.675 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T22:17:20.684 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T22:17:20.684 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T22:17:20.693 DEBUG:teuthology.orchestra.run.smithi118:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T22:17:20.732 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T22:17:20.799 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T22:17:20.838 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T22:17:20.929 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T22:17:20.962 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:17:20.962 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T22:17:21.028 DEBUG:teuthology.orchestra.run.smithi118:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T22:17:21.080 DEBUG:teuthology.orchestra.run.smithi118:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T22:17:21.129 DEBUG:teuthology.orchestra.run.smithi118:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T22:17:21.220 DEBUG:teuthology.orchestra.run.smithi118:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T22:17:21.255 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-13T22:17:21.255 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T22:17:21.321 DEBUG:teuthology.orchestra.run.smithi105:> sudo service rsyslog restart 2024-09-13T22:17:21.323 DEBUG:teuthology.orchestra.run.smithi118:> sudo service rsyslog restart 2024-09-13T22:17:21.370 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:17:21.406 INFO:teuthology.orchestra.run.smithi118.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:17:21.885 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T22:17:21.892 INFO:teuthology.task.internal:Starting timer... 2024-09-13T22:17:21.892 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T22:17:21.904 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T22:17:21.914 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-09-13T22:17:21.915 DEBUG:teuthology.orchestra.run.smithi105:> sudo service auditd rotate 2024-09-13T22:17:21.984 INFO:teuthology.orchestra.run.smithi105.stdout:Rotating logs: 2024-09-13T22:17:21.986 DEBUG:teuthology.orchestra.run.smithi118:> sudo service auditd rotate 2024-09-13T22:17:22.040 INFO:teuthology.orchestra.run.smithi118.stdout:Rotating logs: 2024-09-13T22:17:22.042 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T22:17:22.042 DEBUG:teuthology.orchestra.run.smithi105:> /usr/sbin/getenforce 2024-09-13T22:17:22.066 INFO:teuthology.orchestra.run.smithi105.stdout:Permissive 2024-09-13T22:17:22.066 DEBUG:teuthology.orchestra.run.smithi118:> /usr/sbin/getenforce 2024-09-13T22:17:22.085 INFO:teuthology.orchestra.run.smithi118.stdout:Permissive 2024-09-13T22:17:22.085 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi105.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi118.front.sepia.ceph.com': 'permissive'} 2024-09-13T22:17:22.086 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T22:17:22.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:17:22.134 DEBUG:teuthology.orchestra.run.smithi118:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T22:17:22.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:17:22.161 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T22:17:22.162 DEBUG:teuthology.orchestra.run.smithi105:> sudo /usr/sbin/setenforce permissive 2024-09-13T22:17:22.207 DEBUG:teuthology.orchestra.run.smithi118:> sudo /usr/sbin/setenforce permissive 2024-09-13T22:17:22.247 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T22:17:22.256 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-13T22:17:22.258 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-13T22:17:22.433 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T22:17:22.445 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T22:17:22.446 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi105.front.sepia.ceph.com,smithi118.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T22:21:05.574 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi105.front.sepia.ceph.com'), Remote(name='ubuntu@smithi118.front.sepia.ceph.com')] 2024-09-13T22:21:05.575 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-13T22:21:05.576 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:21:05.662 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-13T22:21:05.748 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-13T22:21:05.748 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-13T22:21:05.749 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T22:21:05.849 DEBUG:teuthology.orchestra.run.smithi118:> true 2024-09-13T22:21:05.933 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-13T22:21:05.933 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T22:21:05.948 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T22:21:05.948 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T22:21:05.948 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:21:05.951 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T22:21:05.951 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:21:05.978 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T22:21:05.993 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T22:21:06.005 INFO:teuthology.orchestra.run.smithi118.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T22:21:06.018 INFO:teuthology.orchestra.run.smithi105.stderr:sudo: ntpd: command not found 2024-09-13T22:21:06.020 INFO:teuthology.orchestra.run.smithi118.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T22:21:06.029 INFO:teuthology.orchestra.run.smithi105.stdout:506 Cannot talk to daemon 2024-09-13T22:21:06.043 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T22:21:06.046 INFO:teuthology.orchestra.run.smithi118.stderr:sudo: ntpd: command not found 2024-09-13T22:21:06.057 INFO:teuthology.orchestra.run.smithi118.stdout:506 Cannot talk to daemon 2024-09-13T22:21:06.058 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T22:21:06.071 INFO:teuthology.orchestra.run.smithi118.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T22:21:06.085 INFO:teuthology.orchestra.run.smithi118.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T22:21:06.127 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:21:06.130 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:21:06.131 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-13T22:21:06.131 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:21:06.131 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:21:06.131 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:21:06.131 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:21:06.150 INFO:teuthology.orchestra.run.smithi118.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:21:06.154 INFO:teuthology.orchestra.run.smithi118.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:21:06.154 INFO:teuthology.orchestra.run.smithi118.stdout:=============================================================================== 2024-09-13T22:21:06.154 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:21:06.155 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:21:06.155 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:21:06.155 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:21:06.156 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T22:21:06.249 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': 'ecab7be58963a5bd03910cd73eef66405d7fdd50'} 2024-09-13T22:21:06.249 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-13T22:21:06.250 INFO:tasks.cephadm:Cluster fsid is 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:06.250 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T22:21:06.250 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-13T22:21:06.250 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi105': '172.21.15.105', 'mon.smithi118': '172.21.15.118'} 2024-09-13T22:21:06.250 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T22:21:06.250 DEBUG:teuthology.orchestra.run.smithi105:> sudo hostname $(hostname -s) 2024-09-13T22:21:06.276 DEBUG:teuthology.orchestra.run.smithi118:> sudo hostname $(hostname -s) 2024-09-13T22:21:06.302 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-13T22:21:06.303 DEBUG:teuthology.orchestra.run.smithi105:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T22:21:06.443 INFO:teuthology.orchestra.run.smithi105.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 22:21 /home/ubuntu/cephtest/cephadm 2024-09-13T22:21:06.443 DEBUG:teuthology.orchestra.run.smithi118:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T22:21:06.605 INFO:teuthology.orchestra.run.smithi118.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 22:21 /home/ubuntu/cephtest/cephadm 2024-09-13T22:21:06.605 DEBUG:teuthology.orchestra.run.smithi105:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T22:21:06.621 DEBUG:teuthology.orchestra.run.smithi118:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T22:21:06.645 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-13T22:21:06.645 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T22:21:06.664 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T22:21:08.251 INFO:teuthology.orchestra.run.smithi105.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T22:21:08.472 INFO:teuthology.orchestra.run.smithi118.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T22:21:19.468 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:21:19.468 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T22:21:19.469 INFO:teuthology.orchestra.run.smithi105.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T22:21:19.469 INFO:teuthology.orchestra.run.smithi105.stdout: "repo_digests": [ 2024-09-13T22:21:19.469 INFO:teuthology.orchestra.run.smithi105.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T22:21:19.469 INFO:teuthology.orchestra.run.smithi105.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T22:21:19.469 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-13T22:21:19.469 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:21:20.983 INFO:teuthology.orchestra.run.smithi118.stdout:{ 2024-09-13T22:21:20.983 INFO:teuthology.orchestra.run.smithi118.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T22:21:20.983 INFO:teuthology.orchestra.run.smithi118.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T22:21:20.983 INFO:teuthology.orchestra.run.smithi118.stdout: "repo_digests": [ 2024-09-13T22:21:20.983 INFO:teuthology.orchestra.run.smithi118.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T22:21:20.984 INFO:teuthology.orchestra.run.smithi118.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T22:21:20.984 INFO:teuthology.orchestra.run.smithi118.stdout: ] 2024-09-13T22:21:20.984 INFO:teuthology.orchestra.run.smithi118.stdout:} 2024-09-13T22:21:21.001 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /etc/ceph 2024-09-13T22:21:21.035 DEBUG:teuthology.orchestra.run.smithi118:> sudo mkdir -p /etc/ceph 2024-09-13T22:21:21.062 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 777 /etc/ceph 2024-09-13T22:21:21.104 DEBUG:teuthology.orchestra.run.smithi118:> sudo chmod 777 /etc/ceph 2024-09-13T22:21:21.128 INFO:tasks.cephadm:Writing seed config... 2024-09-13T22:21:21.129 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T22:21:21.129 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T22:21:21.129 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T22:21:21.129 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T22:21:21.130 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T22:21:21.130 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T22:21:21.130 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T22:21:21.130 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T22:21:21.130 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:21:21.130 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T22:21:21.163 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 782bdfc8-721e-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T22:21:21.163 DEBUG:teuthology.orchestra.run.smithi105:mon.smithi105> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi105.service 2024-09-13T22:21:21.205 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T22:21:21.206 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.105 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:21:21.400 INFO:teuthology.orchestra.run.smithi105.stderr:-------------------------------------------------------------------------------- 2024-09-13T22:21:21.400 INFO:teuthology.orchestra.run.smithi105.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '782bdfc8-721e-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.105', '--skip-admin-label'] 2024-09-13T22:21:21.422 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: 5.2.2 2024-09-13T22:21:21.425 INFO:teuthology.orchestra.run.smithi105.stderr:Verifying podman|docker is present... 2024-09-13T22:21:21.444 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: 5.2.2 2024-09-13T22:21:21.447 INFO:teuthology.orchestra.run.smithi105.stderr:Verifying lvm2 is present... 2024-09-13T22:21:21.447 INFO:teuthology.orchestra.run.smithi105.stderr:Verifying time synchronization is in place... 2024-09-13T22:21:21.454 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T22:21:21.460 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: inactive 2024-09-13T22:21:21.468 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: enabled 2024-09-13T22:21:21.473 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: active 2024-09-13T22:21:21.474 INFO:teuthology.orchestra.run.smithi105.stderr:Unit chronyd.service is enabled and running 2024-09-13T22:21:21.474 INFO:teuthology.orchestra.run.smithi105.stderr:Repeating the final host check... 2024-09-13T22:21:21.493 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: 5.2.2 2024-09-13T22:21:21.496 INFO:teuthology.orchestra.run.smithi105.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-13T22:21:21.496 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl is present 2024-09-13T22:21:21.496 INFO:teuthology.orchestra.run.smithi105.stderr:lvcreate is present 2024-09-13T22:21:21.502 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T22:21:21.508 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: inactive 2024-09-13T22:21:21.515 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: enabled 2024-09-13T22:21:21.521 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: active 2024-09-13T22:21:21.522 INFO:teuthology.orchestra.run.smithi105.stderr:Unit chronyd.service is enabled and running 2024-09-13T22:21:21.522 INFO:teuthology.orchestra.run.smithi105.stderr:Host looks OK 2024-09-13T22:21:21.522 INFO:teuthology.orchestra.run.smithi105.stderr:Cluster fsid: 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:21.522 INFO:teuthology.orchestra.run.smithi105.stderr:Acquiring lock 140589141870480 on /run/cephadm/782bdfc8-721e-11ef-bceb-c7b262605968.lock 2024-09-13T22:21:21.522 INFO:teuthology.orchestra.run.smithi105.stderr:Lock 140589141870480 acquired on /run/cephadm/782bdfc8-721e-11ef-bceb-c7b262605968.lock 2024-09-13T22:21:21.522 INFO:teuthology.orchestra.run.smithi105.stderr:Verifying IP 172.21.15.105 port 3300 ... 2024-09-13T22:21:21.523 INFO:teuthology.orchestra.run.smithi105.stderr:Verifying IP 172.21.15.105 port 6789 ... 2024-09-13T22:21:21.523 INFO:teuthology.orchestra.run.smithi105.stderr:Base mon IP is 172.21.15.105, final addrv is [v2:172.21.15.105:3300,v1:172.21.15.105:6789] 2024-09-13T22:21:21.525 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.105 metric 100 2024-09-13T22:21:21.525 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.105 metric 100 2024-09-13T22:21:21.528 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T22:21:21.528 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-13T22:21:21.528 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-13T22:21:21.530 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T22:21:21.531 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-13T22:21:21.531 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T22:21:21.531 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-13T22:21:21.531 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d429/64 scope link noprefixroute 2024-09-13T22:21:21.531 INFO:teuthology.orchestra.run.smithi105.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T22:21:21.531 INFO:teuthology.orchestra.run.smithi105.stderr:Mon IP `172.21.15.105` is in CIDR network `172.21.0.0/20` 2024-09-13T22:21:21.531 INFO:teuthology.orchestra.run.smithi105.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T22:21:21.532 INFO:teuthology.orchestra.run.smithi105.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T22:21:21.631 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-13T22:21:21.985 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: Getting image source signatures 2024-09-13T22:21:21.985 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-13T22:21:21.986 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-13T22:21:21.986 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-13T22:21:21.986 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-13T22:21:21.986 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-13T22:21:21.986 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T22:21:22.113 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: Writing manifest to image destination 2024-09-13T22:21:22.216 INFO:teuthology.orchestra.run.smithi105.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T22:21:22.601 INFO:teuthology.orchestra.run.smithi105.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T22:21:23.089 INFO:teuthology.orchestra.run.smithi105.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T22:21:23.090 INFO:teuthology.orchestra.run.smithi105.stderr:Extracting ceph user uid/gid from container image... 2024-09-13T22:21:23.406 INFO:teuthology.orchestra.run.smithi105.stderr:stat: 167 167 2024-09-13T22:21:23.878 INFO:teuthology.orchestra.run.smithi105.stderr:Creating initial keys... 2024-09-13T22:21:24.270 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph-authtool: AQDkuuRmKtVyCxAAF1OBNcbPhZRLu8kjJoksuw== 2024-09-13T22:21:25.308 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph-authtool: AQDluuRmXdvVDBAAcfcL8+gC3magzEsreSx+dA== 2024-09-13T22:21:26.280 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph-authtool: AQDmuuRmcHGnDBAAYuibjW7AJXKsVgjKpA/vVw== 2024-09-13T22:21:26.737 INFO:teuthology.orchestra.run.smithi105.stderr:Creating initial monmap... 2024-09-13T22:21:27.152 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T22:21:27.153 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-13T22:21:27.153 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:27.153 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T22:21:27.668 INFO:teuthology.orchestra.run.smithi105.stderr:monmaptool for smithi105 [v2:172.21.15.105:3300,v1:172.21.15.105:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T22:21:27.668 INFO:teuthology.orchestra.run.smithi105.stderr:setting min_mon_release = octopus 2024-09-13T22:21:27.668 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/monmaptool: set fsid to 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:27.668 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T22:21:27.668 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:21:27.668 INFO:teuthology.orchestra.run.smithi105.stderr:Creating mon... 2024-09-13T22:21:28.558 INFO:teuthology.orchestra.run.smithi105.stderr:create mon.smithi105 on 2024-09-13T22:21:28.786 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T22:21:29.086 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-782bdfc8-721e-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-782bdfc8-721e-11ef-bceb-c7b262605968.target. 2024-09-13T22:21:29.086 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-782bdfc8-721e-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-782bdfc8-721e-11ef-bceb-c7b262605968.target. 2024-09-13T22:21:29.701 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: Failed to reset failed state of unit ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi105.service: Unit ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi105.service not loaded. 2024-09-13T22:21:29.711 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: Created symlink /etc/systemd/system/ceph-782bdfc8-721e-11ef-bceb-c7b262605968.target.wants/ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi105.service → /etc/systemd/system/ceph-782bdfc8-721e-11ef-bceb-c7b262605968@.service. 2024-09-13T22:21:30.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:30 smithi105 podman[23339]: 2024-09-13 22:21:30.201236536 +0000 UTC m=+0.016759691 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T22:21:31.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:31 smithi105 podman[23339]: 2024-09-13 22:21:31.341485219 +0000 UTC m=+1.157008408 container create 0ee0fd422be537739ea85333804a0121d9b857fb01ade13dfa65568b99abb54c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_CLEAN=True, release=754, RELEASE=HEAD, GIT_BRANCH=HEAD) 2024-09-13T22:21:33.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:33 smithi105 podman[23339]: 2024-09-13 22:21:33.330933489 +0000 UTC m=+3.146456638 container init 0ee0fd422be537739ea85333804a0121d9b857fb01ade13dfa65568b99abb54c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vendor=Red Hat, Inc., name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8) 2024-09-13T22:21:33.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:33 smithi105 podman[23339]: 2024-09-13 22:21:33.334844944 +0000 UTC m=+3.150368101 container start 0ee0fd422be537739ea85333804a0121d9b857fb01ade13dfa65568b99abb54c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, io.buildah.version=1.19.8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.openshift.expose-services=, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., version=8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, maintainer=Guillaume Abrioux , architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, distribution-scope=public, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T22:21:33.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:33 smithi105 ceph-mon[23368]: mkfs 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:33.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:33 smithi105 ceph-mon[23368]: mon.smithi105 is new leader, mons smithi105 in quorum (ranks 0) 2024-09-13T22:21:34.192 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: disabled 2024-09-13T22:21:34.200 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: inactive 2024-09-13T22:21:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:firewalld.service is not enabled 2024-09-13T22:21:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T22:21:34.202 INFO:teuthology.orchestra.run.smithi105.stderr:Waiting for mon to start... 2024-09-13T22:21:34.202 INFO:teuthology.orchestra.run.smithi105.stderr:Waiting for mon... 2024-09-13T22:21:34.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:34 smithi105 bash[23339]: 0ee0fd422be537739ea85333804a0121d9b857fb01ade13dfa65568b99abb54c 2024-09-13T22:21:34.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:34 smithi105 systemd[1]: Started Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:21:34.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:34 smithi105 ceph-mon[23368]: mon.smithi105 is new leader, mons smithi105 in quorum (ranks 0) 2024-09-13T22:21:34.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:34 smithi105 ceph-mon[23368]: monmap e1: 1 mons at {smithi105=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0]} 2024-09-13T22:21:34.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:34 smithi105 ceph-mon[23368]: fsmap 2024-09-13T22:21:34.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:34 smithi105 ceph-mon[23368]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T22:21:34.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:34 smithi105 ceph-mon[23368]: mgrmap e1: no daemons active 2024-09-13T22:21:37.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:37 smithi105 ceph-mon[23368]: from='client.? 172.21.15.105:0/3397325160' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T22:21:38.210 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: cluster: 2024-09-13T22:21:38.210 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: id: 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:38.210 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-13T22:21:38.210 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: services: 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi105 (age 4s) 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: data: 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: pgs: 2024-09-13T22:21:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:39.636 INFO:teuthology.orchestra.run.smithi105.stderr:mon is available 2024-09-13T22:21:39.637 INFO:teuthology.orchestra.run.smithi105.stderr:Assimilating anything we can from ceph.conf... 2024-09-13T22:21:40.162 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:40.162 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: [global] 2024-09-13T22:21:40.162 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: fsid = 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:40.162 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.105:3300,v1:172.21.15.105:6789] 2024-09-13T22:21:40.162 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T22:21:40.162 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T22:21:40.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T22:21:40.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T22:21:40.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:40.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: [mgr] 2024-09-13T22:21:40.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T22:21:40.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:40.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: [osd] 2024-09-13T22:21:40.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T22:21:40.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T22:21:40.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:40 smithi105 ceph-mon[23368]: from='client.? 172.21.15.105:0/290123794' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T22:21:40.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:40 smithi105 ceph-mon[23368]: from='client.? 172.21.15.105:0/290123794' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T22:21:40.742 INFO:teuthology.orchestra.run.smithi105.stderr:Generating new minimal ceph.conf... 2024-09-13T22:21:41.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:41 smithi105 ceph-mon[23368]: from='client.? 172.21.15.105:0/1065846000' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:21:41.865 INFO:teuthology.orchestra.run.smithi105.stderr:Restarting the monitor... 2024-09-13T22:21:42.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:41 smithi105 systemd[1]: Stopping Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:21:42.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:41 smithi105 bash[23588]: Error: no container with name or ID "ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon.smithi105" found: no such container 2024-09-13T22:21:42.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105[23364]: 2024-09-13T22:21:42.021+0000 7f1fb667f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi105 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:21:42.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105[23364]: 2024-09-13T22:21:42.021+0000 7f1fb667f700 -1 mon.smithi105@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T22:21:42.556 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 podman[23600]: 2024-09-13 22:21:42.279141602 +0000 UTC m=+0.354012662 container died 0ee0fd422be537739ea85333804a0121d9b857fb01ade13dfa65568b99abb54c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, version=8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-type=git, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_CLEAN=True, architecture=x86_64, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-13T22:21:42.823 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 podman[23600]: 2024-09-13 22:21:42.555851767 +0000 UTC m=+0.630722826 container cleanup 0ee0fd422be537739ea85333804a0121d9b857fb01ade13dfa65568b99abb54c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., release=754, com.redhat.component=centos-stream-container, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, version=8, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_CLEAN=True, distribution-scope=public) 2024-09-13T22:21:42.823 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 bash[23600]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105 2024-09-13T22:21:42.823 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 bash[23627]: Error: no container with name or ID "ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon.smithi105" found: no such container 2024-09-13T22:21:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 podman[23614]: 2024-09-13 22:21:42.798447119 +0000 UTC m=+0.522676366 container remove 0ee0fd422be537739ea85333804a0121d9b857fb01ade13dfa65568b99abb54c (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, vendor=Red Hat, Inc., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, distribution-scope=public, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_BRANCH=HEAD, GIT_CLEAN=True, version=8, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8) 2024-09-13T22:21:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi105.service: Deactivated successfully. 2024-09-13T22:21:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 systemd[1]: Stopped Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:21:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:42 smithi105 systemd[1]: Starting Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:21:43.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 podman[23688]: 2024-09-13 22:21:42.93769481 +0000 UTC m=+0.016382093 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T22:21:43.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 podman[23688]: 2024-09-13 22:21:43.054751373 +0000 UTC m=+0.133438642 container create 3e80a4cbe1c04e08a5626d09c28f8701db8d1ce6dd67635876c8e3af74ac06da (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, release=754, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_CLEAN=True, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T22:21:43.338 INFO:teuthology.orchestra.run.smithi105.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-13T22:21:43.461 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 podman[23688]: 2024-09-13 22:21:43.251182322 +0000 UTC m=+0.329869591 container init 3e80a4cbe1c04e08a5626d09c28f8701db8d1ce6dd67635876c8e3af74ac06da (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.k8s.display-name=CentOS Stream 8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, distribution-scope=public, GIT_CLEAN=True, RELEASE=HEAD, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux ) 2024-09-13T22:21:43.461 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 podman[23688]: 2024-09-13 22:21:43.254377829 +0000 UTC m=+0.333065109 container start 3e80a4cbe1c04e08a5626d09c28f8701db8d1ce6dd67635876c8e3af74ac06da (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , version=8, vcs-type=git, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, ceph=True, release=754, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc.) 2024-09-13T22:21:43.461 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:21:43.461 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T22:21:43.461 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: pidfile_write: ignore empty --pid-file 2024-09-13T22:21:43.461 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: load: jerasure load: lrc 2024-09-13T22:21:43.462 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: RocksDB version: 6.15.5 2024-09-13T22:21:43.462 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T22:21:43.462 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Compile date Apr 18 2022 2024-09-13T22:21:43.462 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: DB SUMMARY 2024-09-13T22:21:43.462 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: DB Session ID: MN5U0CH4W0SZXWKN5BQO 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi105/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi105/store.db: 000010.log size: 83529 ; 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.env: 0x5623174c6860 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.fs: Posix File System 2024-09-13T22:21:43.463 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.info_log: 0x5623189e5fa0 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.statistics: (nil) 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.use_fsync: 0 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:21:43.464 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:21:43.465 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:21:43.465 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.db_log_dir: 2024-09-13T22:21:43.465 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi105/store.db 2024-09-13T22:21:43.465 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:21:43.465 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:21:43.465 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.write_buffer_manager: 0x562318ad62d0 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:21:43.466 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.unordered_write: 0 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.row_cache: None 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.wal_filter: None 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.preserve_deletes: 0 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:21:43.467 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:21:43.468 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:21:43.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:21:43.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:21:43.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:21:43.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:21:43.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:21:43.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_open_files: -1 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:21:43.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Compression algorithms supported: 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: kZSTD supported: 0 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi105/store.db/MANIFEST-000009 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.merge_operator: 2024-09-13T22:21:43.471 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_filter: None 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5623189b1d00) 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: index_type: 0 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: data_block_index_type: 0 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: index_shortening: 1 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:21:43.472 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: hash_index_allow_collision: 1 2024-09-13T22:21:43.473 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: checksum: 1 2024-09-13T22:21:43.473 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: no_block_cache: 0 2024-09-13T22:21:43.473 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_cache: 0x562318a1c170 2024-09-13T22:21:43.474 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:21:43.474 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_cache_options: 2024-09-13T22:21:43.474 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: capacity : 536870912 2024-09-13T22:21:43.474 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: num_shard_bits : 4 2024-09-13T22:21:43.474 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: strict_capacity_limit : 0 2024-09-13T22:21:43.474 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:21:43.474 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_cache_compressed: (nil) 2024-09-13T22:21:43.474 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: persistent_cache: (nil) 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_size: 4096 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_size_deviation: 10 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_restart_interval: 16 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: index_block_restart_interval: 1 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: metadata_block_size: 4096 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: partition_filters: 0 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: use_delta_encoding: 1 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: whole_key_filtering: 1 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: verify_compression: 0 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: format_version: 4 2024-09-13T22:21:43.475 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: enable_index_compression: 1 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_align: 0 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compression: NoCompression 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.num_levels: 7 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:21:43.476 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:21:43.477 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:21:43.478 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:21:43.479 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T22:21:43.479 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:21:43.479 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:21:43.479 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T22:21:43.479 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:21:43.479 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:21:43.479 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:21:43.479 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:21:43.479 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.table_properties_collectors: 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:21:43.480 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.ttl: 2592000 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:21:43.481 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi105/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266103280423, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266103281129, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79857, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78102, "index_size": 243, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9930, "raw_average_key_size": 46, "raw_value_size": 72353, "raw_average_value_size": 341, "num_data_blocks": 11, "num_entries": 212, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726266103, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a6aab0dc-3a27-43e8-a274-090fe047471b", "db_session_id": "MN5U0CH4W0SZXWKN5BQO"}} 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266103281435, "job": 1, "event": "recovery_finished"} 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi105/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x562318a02a80 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: DB pointer 0x562318a12000 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T22:21:43.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** DB Stats ** 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: L0 2/0 79.78 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Sum 2/0 79.78 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-13T22:21:43.483 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:21:43.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Cumulative compaction: 0.00 GB write, 26.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Interval compaction: 0.00 GB write, 26.14 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: L0 2/0 79.78 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Sum 2/0 79.78 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-13T22:21:43.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Cumulative compaction: 0.00 GB write, 25.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:21:43.486 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: starting mon.smithi105 rank 0 at public addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] at bind addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi105 fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mon.smithi105@-1(???) e1 preinit fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mon.smithi105@-1(???).mds e1 new map 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mon.smithi105@-1(???).mds e1 print_map 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: e1 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: legacy client fscid: -1 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: No filesystems configured 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mon.smithi105@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T22:21:43.487 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mon.smithi105@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mon.smithi105@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mon.smithi105@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mon.smithi105@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta expand map: {default=false} 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta from 'false' to 'false' 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta expanded map: {default=false} 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta expand map: {default=info} 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta from 'info' to 'info' 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta expanded map: {default=info} 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta expand map: {default=daemon} 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta expanded map: {default=daemon} 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta expand map: {default=debug} 2024-09-13T22:21:43.488 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta from 'debug' to 'debug' 2024-09-13T22:21:43.489 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: expand_channel_meta expanded map: {default=debug} 2024-09-13T22:21:43.489 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 bash[23688]: 3e80a4cbe1c04e08a5626d09c28f8701db8d1ce6dd67635876c8e3af74ac06da 2024-09-13T22:21:43.489 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 systemd[1]: Started Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:21:43.489 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mon.smithi105 is new leader, mons smithi105 in quorum (ranks 0) 2024-09-13T22:21:43.489 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: monmap e1: 1 mons at {smithi105=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0]} 2024-09-13T22:21:43.489 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: fsmap 2024-09-13T22:21:43.489 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T22:21:43.489 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:43 smithi105 ceph-mon[23707]: mgrmap e1: no daemons active 2024-09-13T22:21:45.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:44 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3678623665' entity='client.admin' 2024-09-13T22:21:46.145 INFO:teuthology.orchestra.run.smithi105.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-13T22:21:46.145 INFO:teuthology.orchestra.run.smithi105.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:21:46.145 INFO:teuthology.orchestra.run.smithi105.stderr:Creating mgr... 2024-09-13T22:21:46.145 INFO:teuthology.orchestra.run.smithi105.stderr:Verifying port 9283 ... 2024-09-13T22:21:46.319 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: Failed to reset failed state of unit ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mgr.smithi105.swmmea.service: Unit ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mgr.smithi105.swmmea.service not loaded. 2024-09-13T22:21:46.326 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: Created symlink /etc/systemd/system/ceph-782bdfc8-721e-11ef-bceb-c7b262605968.target.wants/ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mgr.smithi105.swmmea.service → /etc/systemd/system/ceph-782bdfc8-721e-11ef-bceb-c7b262605968@.service. 2024-09-13T22:21:46.966 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: disabled 2024-09-13T22:21:46.973 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: inactive 2024-09-13T22:21:46.974 INFO:teuthology.orchestra.run.smithi105.stderr:firewalld.service is not enabled 2024-09-13T22:21:46.974 INFO:teuthology.orchestra.run.smithi105.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T22:21:46.982 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: disabled 2024-09-13T22:21:46.989 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: inactive 2024-09-13T22:21:46.990 INFO:teuthology.orchestra.run.smithi105.stderr:firewalld.service is not enabled 2024-09-13T22:21:46.990 INFO:teuthology.orchestra.run.smithi105.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-13T22:21:46.990 INFO:teuthology.orchestra.run.smithi105.stderr:Waiting for mgr to start... 2024-09-13T22:21:46.990 INFO:teuthology.orchestra.run.smithi105.stderr:Waiting for mgr... 2024-09-13T22:21:47.542 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:47.545 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: { 2024-09-13T22:21:47.545 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "fsid": "782bdfc8-721e-11ef-bceb-c7b262605968", 2024-09-13T22:21:47.545 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "health": { 2024-09-13T22:21:47.546 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T22:21:47.546 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T22:21:47.546 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T22:21:47.546 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:47.546 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T22:21:47.546 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 0 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "smithi105" 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum_age": 4, 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T22:21:47.547 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T22:21:47.548 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:47.548 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T22:21:47.548 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:47.548 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T22:21:47.548 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T22:21:47.548 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T22:21:47.548 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T22:21:47.552 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T22:21:47.552 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T22:21:47.552 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:47.552 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T22:21:47.552 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T22:21:47.552 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:47.553 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "available": false, 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "iostat", 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "nfs", 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "restful" 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:47.554 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "modified": "2024-09-13T22:21:33.377513+0000", 2024-09-13T22:21:47.555 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:21:47.555 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:47.555 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T22:21:47.555 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: } 2024-09-13T22:21:47.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:47 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2046906754' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:21:48.078 INFO:teuthology.orchestra.run.smithi105.stderr:mgr not available, waiting (1/15)... 2024-09-13T22:21:50.606 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:50.607 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: { 2024-09-13T22:21:50.607 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "fsid": "782bdfc8-721e-11ef-bceb-c7b262605968", 2024-09-13T22:21:50.607 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "health": { 2024-09-13T22:21:50.607 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T22:21:50.607 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T22:21:50.607 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T22:21:50.607 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:50.608 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T22:21:50.608 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T22:21:50.608 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 0 2024-09-13T22:21:50.608 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:50.608 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T22:21:50.608 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "smithi105" 2024-09-13T22:21:50.608 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:50.609 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum_age": 7, 2024-09-13T22:21:50.609 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T22:21:50.609 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:50.609 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T22:21:50.609 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T22:21:50.609 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:50.609 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T22:21:50.609 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:50.610 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T22:21:50.610 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T22:21:50.610 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T22:21:50.610 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T22:21:50.610 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T22:21:50.610 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T22:21:50.610 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:50.610 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T22:21:50.611 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T22:21:50.611 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T22:21:50.611 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T22:21:50.611 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T22:21:50.611 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T22:21:50.611 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T22:21:50.611 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T22:21:50.611 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T22:21:50.612 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:50.612 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T22:21:50.612 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:50.612 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T22:21:50.612 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T22:21:50.612 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:50.612 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T22:21:50.612 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "available": false, 2024-09-13T22:21:50.613 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T22:21:50.613 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T22:21:50.613 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "iostat", 2024-09-13T22:21:50.613 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "nfs", 2024-09-13T22:21:50.613 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "restful" 2024-09-13T22:21:50.613 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:50.613 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:21:50.613 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:50.614 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T22:21:50.614 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:50.614 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "modified": "2024-09-13T22:21:33.377513+0000", 2024-09-13T22:21:50.614 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:21:50.614 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:50.614 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T22:21:50.614 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: } 2024-09-13T22:21:50.816 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:50 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/556623956' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:21:51.210 INFO:teuthology.orchestra.run.smithi105.stderr:mgr not available, waiting (2/15)... 2024-09-13T22:21:53.333 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: Activating manager daemon smithi105.swmmea 2024-09-13T22:21:53.333 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: mgrmap e2: smithi105.swmmea(active, starting, since 0.0029093s) 2024-09-13T22:21:53.333 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:21:53.333 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:21:53.333 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:21:53.333 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:21:53.333 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:21:53.333 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: Manager daemon smithi105.swmmea is now available 2024-09-13T22:21:53.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:21:53.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:21:53.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' 2024-09-13T22:21:53.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' 2024-09-13T22:21:53.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:53 smithi105 ceph-mon[23707]: from='mgr.14100 172.21.15.105:0/1710328064' entity='mgr.smithi105.swmmea' 2024-09-13T22:21:53.696 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: { 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "fsid": "782bdfc8-721e-11ef-bceb-c7b262605968", 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "health": { 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 0 2024-09-13T22:21:53.697 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "smithi105" 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum_age": 10, 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T22:21:53.698 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T22:21:53.699 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T22:21:53.703 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "available": false, 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T22:21:53.704 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "iostat", 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "nfs", 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "restful" 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "modified": "2024-09-13T22:21:33.377513+0000", 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T22:21:53.705 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: } 2024-09-13T22:21:54.257 INFO:teuthology.orchestra.run.smithi105.stderr:mgr not available, waiting (3/15)... 2024-09-13T22:21:54.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:54 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3631581313' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:21:54.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:54 smithi105 ceph-mon[23707]: mgrmap e3: smithi105.swmmea(active, since 1.00543s) 2024-09-13T22:21:56.590 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:56 smithi105 ceph-mon[23707]: mgrmap e4: smithi105.swmmea(active, since 2s) 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: { 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "fsid": "782bdfc8-721e-11ef-bceb-c7b262605968", 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "health": { 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T22:21:56.936 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 0 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "smithi105" 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "quorum_age": 13, 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:56.937 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T22:21:56.938 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T22:21:56.938 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T22:21:56.938 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T22:21:56.941 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T22:21:56.941 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T22:21:56.941 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:56.941 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T22:21:56.941 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T22:21:56.941 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T22:21:56.941 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T22:21:56.941 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T22:21:56.941 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "available": true, 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T22:21:56.942 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "iostat", 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "nfs", 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "restful" 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ], 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "modified": "2024-09-13T22:21:33.377513+0000", 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "services": {} 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: }, 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T22:21:56.943 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: } 2024-09-13T22:21:57.529 INFO:teuthology.orchestra.run.smithi105.stderr:mgr is available 2024-09-13T22:21:57.636 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:57 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/157866883' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T22:21:58.146 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: [global] 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: fsid = 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: [mgr] 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: [osd] 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T22:21:58.147 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T22:21:58.633 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:58 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1750753830' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T22:21:58.635 INFO:teuthology.orchestra.run.smithi105.stderr:Enabling cephadm module... 2024-09-13T22:21:59.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:21:59 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2298463742' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T22:22:00.566 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:00 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2298463742' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T22:22:00.567 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:00 smithi105 ceph-mon[23707]: mgrmap e5: smithi105.swmmea(active, since 6s) 2024-09-13T22:22:00.662 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: { 2024-09-13T22:22:00.662 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-13T22:22:00.662 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "available": true, 2024-09-13T22:22:00.662 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "active_name": "smithi105.swmmea", 2024-09-13T22:22:00.662 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T22:22:00.662 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: } 2024-09-13T22:22:01.316 INFO:teuthology.orchestra.run.smithi105.stderr:Waiting for the mgr to restart... 2024-09-13T22:22:01.316 INFO:teuthology.orchestra.run.smithi105.stderr:Waiting for mgr epoch 5... 2024-09-13T22:22:01.588 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:01 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3163253658' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T22:22:05.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:05 smithi105 ceph-mon[23707]: Active manager daemon smithi105.swmmea restarted 2024-09-13T22:22:05.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:05 smithi105 ceph-mon[23707]: Activating manager daemon smithi105.swmmea 2024-09-13T22:22:05.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:05 smithi105 ceph-mon[23707]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T22:22:06.501 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: { 2024-09-13T22:22:06.502 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-13T22:22:06.502 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T22:22:06.502 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: } 2024-09-13T22:22:06.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: mgrmap e6: smithi105.swmmea(active, starting, since 0.0523814s) 2024-09-13T22:22:06.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:22:06.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:22:06.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:22:06.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:22:06.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:22:06.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: Manager daemon smithi105.swmmea is now available 2024-09-13T22:22:06.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: [13/Sep/2024:22:22:05] ENGINE Bus STARTING 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: [13/Sep/2024:22:22:06] ENGINE Serving on https://172.21.15.105:7150 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: [13/Sep/2024:22:22:06] ENGINE Bus STARTED 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:06.812 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:06 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:22:07.113 INFO:teuthology.orchestra.run.smithi105.stderr:mgr epoch 5 is available 2024-09-13T22:22:07.113 INFO:teuthology.orchestra.run.smithi105.stderr:Setting orchestrator backend to cephadm... 2024-09-13T22:22:07.746 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:07 smithi105 ceph-mon[23707]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T22:22:07.747 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:07 smithi105 ceph-mon[23707]: mgrmap e7: smithi105.swmmea(active, since 1.05941s) 2024-09-13T22:22:07.747 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:07 smithi105 ceph-mon[23707]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T22:22:08.610 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:08 smithi105 ceph-mon[23707]: mgrmap e8: smithi105.swmmea(active, since 2s) 2024-09-13T22:22:08.610 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:08 smithi105 ceph-mon[23707]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:08.610 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:08 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:08.610 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:08 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:22:08.885 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: value unchanged 2024-09-13T22:22:09.424 INFO:teuthology.orchestra.run.smithi105.stderr:Generating ssh key... 2024-09-13T22:22:09.783 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:09 smithi105 ceph-mon[23707]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:11.253 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:11 smithi105 ceph-mon[23707]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:11.253 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:11 smithi105 ceph-mon[23707]: Generating ssh key... 2024-09-13T22:22:11.253 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:11 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:11.253 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:11 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:11.253 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpvY+I/YX63XwSMI55EQ8alJndqOCvHTKkKMFkRJcNp2cw67cmMZMC4jmVTNDtjcLtpQNEWAo/HK8LWvR0x5Mq1ar+7zvliiJX5SrI37xdMWsOG554+FILth+qIrtCzKDVSqS8gDthHL2/G4hj+/+FoCTOgfUrWohXPXAtWHk5ASGSQ03bjBIXfP64VIgYqmN11Zx1vKBPUlOrm09eVDM8rMw5C9MZ4pZyVejflAMJJjxReCPiIhQdQ0dPORJlWE3f081XBAcnpWSSRWhMwVB5pjfhjK82cjDDMriL88+DV5FmsH17UXcv2T/wrvlU7dZRdaeolf5PonleCWJJ12x5mMxoXyxWTHIbIaVoBKjGeBqYEugY+I/X7q39lxrfjL5vQiHtIEz2VZyVEEKRv1oJY6HpndZNdfFPqRzgliy1Zf9wLnhusqzLUyQGMYXHJ1MfTC9sq0oii9W6fpzzqPskuRGywo1oJqzuKaH3do/TzgO455PIovhtFbNzV1QEtys= ceph-782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:22:11.853 INFO:teuthology.orchestra.run.smithi105.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T22:22:11.854 INFO:teuthology.orchestra.run.smithi105.stderr:Adding key to root@localhost authorized_keys... 2024-09-13T22:22:11.858 INFO:teuthology.orchestra.run.smithi105.stderr:Adding host smithi105... 2024-09-13T22:22:12.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:12 smithi105 ceph-mon[23707]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:13.329 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: Added host 'smithi105' with addr '172.21.15.105' 2024-09-13T22:22:13.911 INFO:teuthology.orchestra.run.smithi105.stderr:Deploying mon service with default placement... 2024-09-13T22:22:14.538 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:14 smithi105 ceph-mon[23707]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi105", "addr": "172.21.15.105", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:14.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:14 smithi105 ceph-mon[23707]: Deploying cephadm binary to smithi105 2024-09-13T22:22:14.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:14 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:14.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:14 smithi105 ceph-mon[23707]: Added host smithi105 2024-09-13T22:22:14.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:14 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:22:14.539 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-13T22:22:15.252 INFO:teuthology.orchestra.run.smithi105.stderr:Deploying mgr service with default placement... 2024-09-13T22:22:15.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:15 smithi105 ceph-mon[23707]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:15.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:15 smithi105 ceph-mon[23707]: Saving service mon spec with placement count:5 2024-09-13T22:22:15.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:15 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:15.914 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-13T22:22:17.329 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:16 smithi105 ceph-mon[23707]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:17.329 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:16 smithi105 ceph-mon[23707]: Saving service mgr spec with placement count:2 2024-09-13T22:22:17.329 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:16 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:17.332 INFO:teuthology.orchestra.run.smithi105.stderr:Deploying crash service with default placement... 2024-09-13T22:22:18.044 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-13T22:22:18.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:17 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:18.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:17 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:18.874 INFO:teuthology.orchestra.run.smithi105.stderr:Deploying prometheus service with default placement... 2024-09-13T22:22:19.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:19 smithi105 ceph-mon[23707]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:19.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:19 smithi105 ceph-mon[23707]: Saving service crash spec with placement * 2024-09-13T22:22:19.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:19 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:19.468 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-13T22:22:20.258 INFO:teuthology.orchestra.run.smithi105.stderr:Deploying grafana service with default placement... 2024-09-13T22:22:20.513 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:20 smithi105 ceph-mon[23707]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:20.513 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:20 smithi105 ceph-mon[23707]: Saving service prometheus spec with placement count:1 2024-09-13T22:22:20.513 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:20 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:20.513 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:20 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:20.951 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-13T22:22:21.624 INFO:teuthology.orchestra.run.smithi105.stderr:Deploying node-exporter service with default placement... 2024-09-13T22:22:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:21 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:21 smithi105 ceph-mon[23707]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:21 smithi105 ceph-mon[23707]: Saving service grafana spec with placement count:1 2024-09-13T22:22:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:21 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:22.272 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-13T22:22:23.583 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:23 smithi105 ceph-mon[23707]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:23.583 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:23 smithi105 ceph-mon[23707]: Saving service node-exporter spec with placement * 2024-09-13T22:22:23.583 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:23 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:23.586 INFO:teuthology.orchestra.run.smithi105.stderr:Deploying alertmanager service with default placement... 2024-09-13T22:22:24.276 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-13T22:22:25.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:25 smithi105 ceph-mon[23707]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:25.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:25 smithi105 ceph-mon[23707]: Saving service alertmanager spec with placement count:1 2024-09-13T22:22:25.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:25 smithi105 ceph-mon[23707]: from='mgr.14120 172.21.15.105:0/1530774021' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:26.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:26 smithi105 ceph-mon[23707]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:22:26.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:26 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/929509278' entity='client.admin' 2024-09-13T22:22:27.514 INFO:teuthology.orchestra.run.smithi105.stderr:Enabling the dashboard module... 2024-09-13T22:22:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:27 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1647376837' entity='client.admin' 2024-09-13T22:22:29.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:28 smithi105 ceph-mon[23707]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:22:29.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:28 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2099412445' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T22:22:30.110 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:29 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2099412445' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T22:22:30.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:29 smithi105 ceph-mon[23707]: mgrmap e9: smithi105.swmmea(active, since 23s) 2024-09-13T22:22:30.245 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: { 2024-09-13T22:22:30.245 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-13T22:22:30.245 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "available": true, 2024-09-13T22:22:30.245 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "active_name": "smithi105.swmmea", 2024-09-13T22:22:30.245 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T22:22:30.245 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: } 2024-09-13T22:22:30.828 INFO:teuthology.orchestra.run.smithi105.stderr:Waiting for the mgr to restart... 2024-09-13T22:22:30.828 INFO:teuthology.orchestra.run.smithi105.stderr:Waiting for mgr epoch 9... 2024-09-13T22:22:31.100 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:30 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2291937750' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T22:22:36.042 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: { 2024-09-13T22:22:36.042 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-13T22:22:36.042 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T22:22:36.042 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: } 2024-09-13T22:22:36.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: Active manager daemon smithi105.swmmea restarted 2024-09-13T22:22:36.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: Activating manager daemon smithi105.swmmea 2024-09-13T22:22:36.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T22:22:36.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: mgrmap e10: smithi105.swmmea(active, starting, since 0.0535663s) 2024-09-13T22:22:36.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: Manager daemon smithi105.swmmea is now available 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: [13/Sep/2024:22:22:35] ENGINE Bus STARTING 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: [13/Sep/2024:22:22:35] ENGINE Serving on https://172.21.15.105:7150 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: [13/Sep/2024:22:22:35] ENGINE Bus STARTED 2024-09-13T22:22:36.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:35 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:36.947 INFO:teuthology.orchestra.run.smithi105.stderr:mgr epoch 9 is available 2024-09-13T22:22:36.947 INFO:teuthology.orchestra.run.smithi105.stderr:Generating a dashboard self-signed certificate... 2024-09-13T22:22:37.281 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:37 smithi105 ceph-mon[23707]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T22:22:37.281 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:37 smithi105 ceph-mon[23707]: mgrmap e11: smithi105.swmmea(active, since 1.05604s) 2024-09-13T22:22:37.281 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:37 smithi105 ceph-mon[23707]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T22:22:37.666 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-13T22:22:38.318 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:38 smithi105 ceph-mon[23707]: mgrmap e12: smithi105.swmmea(active, since 2s) 2024-09-13T22:22:38.318 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:38 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:38.319 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:38 smithi105 ceph-mon[23707]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:38.319 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:38 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:38.319 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:38 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:38.320 INFO:teuthology.orchestra.run.smithi105.stderr:Creating initial admin user... 2024-09-13T22:22:39.163 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$5ExEqQXEVE953CMjVCYhD.SPQwadnlMkPJLkRw9XhvCXOhdzKoS3C", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726266159, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T22:22:39.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:39 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:39.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:39 smithi105 ceph-mon[23707]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:39.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:39 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:40.297 INFO:teuthology.orchestra.run.smithi105.stderr:Fetching dashboard port number... 2024-09-13T22:22:41.384 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: 8443 2024-09-13T22:22:42.092 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: disabled 2024-09-13T22:22:42.099 INFO:teuthology.orchestra.run.smithi105.stderr:systemctl: inactive 2024-09-13T22:22:42.099 INFO:teuthology.orchestra.run.smithi105.stderr:firewalld.service is not enabled 2024-09-13T22:22:42.099 INFO:teuthology.orchestra.run.smithi105.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T22:22:42.101 INFO:teuthology.orchestra.run.smithi105.stderr:Ceph Dashboard is now available at: 2024-09-13T22:22:42.101 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:22:42.101 INFO:teuthology.orchestra.run.smithi105.stderr: URL: https://smithi105.front.sepia.ceph.com:8443/ 2024-09-13T22:22:42.101 INFO:teuthology.orchestra.run.smithi105.stderr: User: admin 2024-09-13T22:22:42.102 INFO:teuthology.orchestra.run.smithi105.stderr: Password: c241xsy6mt 2024-09-13T22:22:42.102 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:22:42.102 INFO:teuthology.orchestra.run.smithi105.stderr:Enabling autotune for osd_memory_target 2024-09-13T22:22:42.357 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:42 smithi105 ceph-mon[23707]: mgrmap e13: smithi105.swmmea(active, since 6s) 2024-09-13T22:22:42.357 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:42 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/162229676' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T22:22:44.128 INFO:teuthology.orchestra.run.smithi105.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-13T22:22:44.762 INFO:teuthology.orchestra.run.smithi105.stderr:You can access the Ceph CLI with: 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr: ceph telemetry on 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr:For more information see: 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:22:44.763 INFO:teuthology.orchestra.run.smithi105.stderr:Bootstrap complete. 2024-09-13T22:22:44.794 INFO:tasks.cephadm:Fetching config... 2024-09-13T22:22:44.795 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:22:44.795 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T22:22:44.811 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T22:22:44.811 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:22:44.811 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T22:22:44.868 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T22:22:44.868 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:22:44.868 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/keyring of=/dev/stdout 2024-09-13T22:22:44.932 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T22:22:44.933 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:22:44.933 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T22:22:44.989 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T22:22:44.989 DEBUG:teuthology.orchestra.run.smithi105:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpvY+I/YX63XwSMI55EQ8alJndqOCvHTKkKMFkRJcNp2cw67cmMZMC4jmVTNDtjcLtpQNEWAo/HK8LWvR0x5Mq1ar+7zvliiJX5SrI37xdMWsOG554+FILth+qIrtCzKDVSqS8gDthHL2/G4hj+/+FoCTOgfUrWohXPXAtWHk5ASGSQ03bjBIXfP64VIgYqmN11Zx1vKBPUlOrm09eVDM8rMw5C9MZ4pZyVejflAMJJjxReCPiIhQdQ0dPORJlWE3f081XBAcnpWSSRWhMwVB5pjfhjK82cjDDMriL88+DV5FmsH17UXcv2T/wrvlU7dZRdaeolf5PonleCWJJ12x5mMxoXyxWTHIbIaVoBKjGeBqYEugY+I/X7q39lxrfjL5vQiHtIEz2VZyVEEKRv1oJY6HpndZNdfFPqRzgliy1Zf9wLnhusqzLUyQGMYXHJ1MfTC9sq0oii9W6fpzzqPskuRGywo1oJqzuKaH3do/TzgO455PIovhtFbNzV1QEtys= ceph-782bdfc8-721e-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T22:22:45.064 INFO:teuthology.orchestra.run.smithi105.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpvY+I/YX63XwSMI55EQ8alJndqOCvHTKkKMFkRJcNp2cw67cmMZMC4jmVTNDtjcLtpQNEWAo/HK8LWvR0x5Mq1ar+7zvliiJX5SrI37xdMWsOG554+FILth+qIrtCzKDVSqS8gDthHL2/G4hj+/+FoCTOgfUrWohXPXAtWHk5ASGSQ03bjBIXfP64VIgYqmN11Zx1vKBPUlOrm09eVDM8rMw5C9MZ4pZyVejflAMJJjxReCPiIhQdQ0dPORJlWE3f081XBAcnpWSSRWhMwVB5pjfhjK82cjDDMriL88+DV5FmsH17UXcv2T/wrvlU7dZRdaeolf5PonleCWJJ12x5mMxoXyxWTHIbIaVoBKjGeBqYEugY+I/X7q39lxrfjL5vQiHtIEz2VZyVEEKRv1oJY6HpndZNdfFPqRzgliy1Zf9wLnhusqzLUyQGMYXHJ1MfTC9sq0oii9W6fpzzqPskuRGywo1oJqzuKaH3do/TzgO455PIovhtFbNzV1QEtys= ceph-782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:22:45.076 DEBUG:teuthology.orchestra.run.smithi118:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpvY+I/YX63XwSMI55EQ8alJndqOCvHTKkKMFkRJcNp2cw67cmMZMC4jmVTNDtjcLtpQNEWAo/HK8LWvR0x5Mq1ar+7zvliiJX5SrI37xdMWsOG554+FILth+qIrtCzKDVSqS8gDthHL2/G4hj+/+FoCTOgfUrWohXPXAtWHk5ASGSQ03bjBIXfP64VIgYqmN11Zx1vKBPUlOrm09eVDM8rMw5C9MZ4pZyVejflAMJJjxReCPiIhQdQ0dPORJlWE3f081XBAcnpWSSRWhMwVB5pjfhjK82cjDDMriL88+DV5FmsH17UXcv2T/wrvlU7dZRdaeolf5PonleCWJJ12x5mMxoXyxWTHIbIaVoBKjGeBqYEugY+I/X7q39lxrfjL5vQiHtIEz2VZyVEEKRv1oJY6HpndZNdfFPqRzgliy1Zf9wLnhusqzLUyQGMYXHJ1MfTC9sq0oii9W6fpzzqPskuRGywo1oJqzuKaH3do/TzgO455PIovhtFbNzV1QEtys= ceph-782bdfc8-721e-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T22:22:45.141 INFO:teuthology.orchestra.run.smithi118.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpvY+I/YX63XwSMI55EQ8alJndqOCvHTKkKMFkRJcNp2cw67cmMZMC4jmVTNDtjcLtpQNEWAo/HK8LWvR0x5Mq1ar+7zvliiJX5SrI37xdMWsOG554+FILth+qIrtCzKDVSqS8gDthHL2/G4hj+/+FoCTOgfUrWohXPXAtWHk5ASGSQ03bjBIXfP64VIgYqmN11Zx1vKBPUlOrm09eVDM8rMw5C9MZ4pZyVejflAMJJjxReCPiIhQdQ0dPORJlWE3f081XBAcnpWSSRWhMwVB5pjfhjK82cjDDMriL88+DV5FmsH17UXcv2T/wrvlU7dZRdaeolf5PonleCWJJ12x5mMxoXyxWTHIbIaVoBKjGeBqYEugY+I/X7q39lxrfjL5vQiHtIEz2VZyVEEKRv1oJY6HpndZNdfFPqRzgliy1Zf9wLnhusqzLUyQGMYXHJ1MfTC9sq0oii9W6fpzzqPskuRGywo1oJqzuKaH3do/TzgO455PIovhtFbNzV1QEtys= ceph-782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:22:45.154 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T22:22:45.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:45 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3096983366' entity='client.admin' 2024-09-13T22:22:48.609 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T22:22:48.610 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T22:22:49.088 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:48 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2459914253' entity='client.admin' 2024-09-13T22:22:50.453 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:50 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:50.453 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:50 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd/host:smithi105", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:22:50.453 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:50 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:50.453 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:50 smithi105 ceph-mon[23707]: Deploying daemon alertmanager.smithi105 on smithi105 2024-09-13T22:22:51.800 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi118 2024-09-13T22:22:51.800 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-13T22:22:51.800 DEBUG:teuthology.orchestra.run.smithi118:> dd of=/etc/ceph/ceph.conf 2024-09-13T22:22:51.819 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-13T22:22:51.819 DEBUG:teuthology.orchestra.run.smithi118:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:22:51.877 INFO:tasks.cephadm:Adding host smithi118 to orchestrator... 2024-09-13T22:22:51.877 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch host add smithi118 2024-09-13T22:22:52.182 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:52 smithi105 ceph-mon[23707]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:52.182 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:52 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:55.919 INFO:teuthology.orchestra.run.smithi105.stdout:Added host 'smithi118' with addr '172.21.15.118' 2024-09-13T22:22:56.528 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T22:22:57.190 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:56 smithi105 ceph-mon[23707]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi118", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:22:57.191 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:56 smithi105 ceph-mon[23707]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:22:57.191 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:56 smithi105 ceph-mon[23707]: Deploying cephadm binary to smithi118 2024-09-13T22:22:57.191 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:56 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:22:57.191 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:56 smithi105 ceph-mon[23707]: Added host smithi118 2024-09-13T22:22:59.371 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:22:58 smithi105 ceph-mon[23707]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:00.307 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:23:00.307 INFO:teuthology.orchestra.run.smithi105.stdout:[{"addr": "172.21.15.105", "hostname": "smithi105", "labels": [], "status": ""}, {"addr": "172.21.15.118", "hostname": "smithi118", "labels": [], "status": ""}] 2024-09-13T22:23:01.291 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:00 smithi105 ceph-mon[23707]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:01.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:00 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:01.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:00 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi105", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:23:01.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:00 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi105", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T22:23:01.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:00 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:01.313 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T22:23:01.313 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T22:23:02.197 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:01 smithi105 ceph-mon[23707]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:23:02.197 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:01 smithi105 ceph-mon[23707]: Deploying daemon crash.smithi105 on smithi105 2024-09-13T22:23:03.191 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:02 smithi105 ceph-mon[23707]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:05.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:04 smithi105 ceph-mon[23707]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:05.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:04 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:05.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:04 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:05.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:04 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:05.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:04 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:23:05.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:04 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:05.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:04 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3251333457' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T22:23:05.402 INFO:teuthology.orchestra.run.smithi105.stderr:adjusted tunables profile to default 2024-09-13T22:23:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:05 smithi105 ceph-mon[23707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:23:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:05 smithi105 ceph-mon[23707]: Deploying daemon grafana.smithi105 on smithi105 2024-09-13T22:23:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:05 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:05 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3251333457' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T22:23:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:05 smithi105 ceph-mon[23707]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T22:23:06.176 INFO:tasks.cephadm:Adding mon.smithi105 on smithi105 2024-09-13T22:23:06.176 INFO:tasks.cephadm:Adding mon.smithi118 on smithi118 2024-09-13T22:23:06.177 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi105:172.21.15.105=smithi105;smithi118:172.21.15.118=smithi118' 2024-09-13T22:23:07.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:06 smithi105 ceph-mon[23707]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:08.135 INFO:teuthology.orchestra.run.smithi118.stdout:Scheduled mon update... 2024-09-13T22:23:08.669 DEBUG:teuthology.orchestra.run.smithi118:mon.smithi118> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi118.service 2024-09-13T22:23:08.672 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:08.672 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:09.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:08 smithi105 ceph-mon[23707]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:09.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:08 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:10.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:09 smithi105 ceph-mon[23707]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi105:172.21.15.105=smithi105;smithi118:172.21.15.118=smithi118", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:23:10.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:09 smithi105 ceph-mon[23707]: Saving service mon spec with placement smithi105:172.21.15.105=smithi105;smithi118:172.21.15.118=smithi118;count:2 2024-09-13T22:23:10.472 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:10.472 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:21:27.083377Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:23:10.474 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-13T22:23:11.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:10 smithi105 ceph-mon[23707]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:11.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:10 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/1030761713' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:12.051 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:12.052 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:13.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:12 smithi105 ceph-mon[23707]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:13.947 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:13.948 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:21:27.083377Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:23:13.950 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-13T22:23:15.085 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:14 smithi105 ceph-mon[23707]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:15.085 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:14 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/3997076632' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:15.085 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:14 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:15.542 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:15.542 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:16.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:15 smithi105 ceph-mon[23707]: Deploying daemon node-exporter.smithi105 on smithi105 2024-09-13T22:23:16.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:15 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:17.354 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:16 smithi105 ceph-mon[23707]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:17.459 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:17.459 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:21:27.083377Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:23:17.465 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-13T22:23:18.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:17 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/233612315' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:18.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:17 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:18.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:17 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T22:23:19.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:18 smithi105 ceph-mon[23707]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:19.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:18 smithi105 ceph-mon[23707]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T22:23:19.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:18 smithi105 ceph-mon[23707]: mgrmap e14: smithi105.swmmea(active, since 43s) 2024-09-13T22:23:19.157 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:19.157 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:21.009 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:21.009 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:21:27.083377Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:23:21.010 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-13T22:23:21.348 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:21 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/1916087297' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:22.633 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:22.633 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:24.501 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:24.501 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:21:27.083377Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:23:24.503 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-13T22:23:24.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:24 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/473855600' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:25.849 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:25 smithi105 ceph-mon[23707]: Active manager daemon smithi105.swmmea restarted 2024-09-13T22:23:25.849 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:25 smithi105 ceph-mon[23707]: Activating manager daemon smithi105.swmmea 2024-09-13T22:23:25.850 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:25 smithi105 ceph-mon[23707]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T22:23:26.012 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:26.013 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:26.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: mgrmap e15: smithi105.swmmea(active, starting, since 0.769983s) 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: Manager daemon smithi105.swmmea is now available 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:23:26.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:23:26.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:26.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:27.878 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:27.878 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:21:27.083377Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:23:27.880 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-13T22:23:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:27 smithi105 ceph-mon[23707]: [13/Sep/2024:22:23:26] ENGINE Bus STARTING 2024-09-13T22:23:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:27 smithi105 ceph-mon[23707]: [13/Sep/2024:22:23:26] ENGINE Serving on https://172.21.15.105:7150 2024-09-13T22:23:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:27 smithi105 ceph-mon[23707]: [13/Sep/2024:22:23:26] ENGINE Bus STARTED 2024-09-13T22:23:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:27 smithi105 ceph-mon[23707]: mgrmap e16: smithi105.swmmea(active, since 1.77429s) 2024-09-13T22:23:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:27 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:28.654 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:28 smithi105 ceph-mon[23707]: mgrmap e17: smithi105.swmmea(active, since 2s) 2024-09-13T22:23:28.654 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:28 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:28.654 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:28 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/3676715197' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:29.452 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:29.452 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:30.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:29 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:30.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:29 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:30.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:29 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd/host:smithi105", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:23:30.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:29 smithi105 ceph-mon[23707]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:23:30.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:29 smithi105 ceph-mon[23707]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:23:30.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:29 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:32.155 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:32.155 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:21:27.083377Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:23:32.157 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-13T22:23:32.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:32.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:32.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd/host:smithi118", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:23:32.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:32.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:23:32.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T22:23:32.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:32.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:32 smithi105 ceph-mon[23707]: Deploying daemon crash.smithi118 on smithi118 2024-09-13T22:23:33.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:33 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/751919740' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:34.595 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:34.595 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:35.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:35 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:35.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:35 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:23:35.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:35 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T22:23:35.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:35 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:23:35.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:35 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:35.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:35 smithi105 ceph-mon[23707]: Deploying daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:23:36.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:36 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:37.762 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:37.763 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:21:27.083377Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:23:37.765 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-13T22:23:38.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:37 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/2460823911' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:38.182 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:37 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:38.182 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:37 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:23:38.182 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:37 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:39.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:38 smithi105 ceph-mon[23707]: Deploying daemon mon.smithi118 on smithi118 2024-09-13T22:23:39.406 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:39.406 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:41.401 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:41.401 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:21:27.083377Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T22:23:41.403 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-13T22:23:41.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:41 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:41.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:41 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/2652514894' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:42 smithi118 systemd[1]: Starting Ceph mon.smithi118 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:23:43.163 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T22:23:43.163 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T22:23:43.429 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 podman[25668]: 2024-09-13 22:23:43.075971662 +0000 UTC m=+0.016756938 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-13T22:23:43.429 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 podman[25668]: 2024-09-13 22:23:43.200918591 +0000 UTC m=+0.141703860 container create 17b16ac8afb2e81435a3ccd15b75552a8f5e1c5367f18162469ff95bd6280c01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, name=centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, release=754, RELEASE=HEAD, GIT_BRANCH=HEAD, version=8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T22:23:43.430 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 podman[25668]: 2024-09-13 22:23:43.428728487 +0000 UTC m=+0.369513769 container init 17b16ac8afb2e81435a3ccd15b75552a8f5e1c5367f18162469ff95bd6280c01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, RELEASE=HEAD, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, ceph=True, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_CLEAN=True, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream) 2024-09-13T22:23:43.705 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 podman[25668]: 2024-09-13 22:23:43.433291929 +0000 UTC m=+0.374077198 container start 17b16ac8afb2e81435a3ccd15b75552a8f5e1c5367f18162469ff95bd6280c01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, distribution-scope=public, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=centos-stream, release=754, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, ceph=True, io.openshift.tags=base centos centos-stream, vcs-type=git, version=8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pidfile_write: ignore empty --pid-file 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: load: jerasure load: lrc 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: RocksDB version: 6.15.5 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Compile date Apr 18 2022 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: DB SUMMARY 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: DB Session ID: 2V2HIDCI6W3KVDN1JLKC 2024-09-13T22:23:43.706 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi118/store.db dir, Total Num: 0, files: 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi118/store.db: 000004.log size: 511 ; 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.env: 0x56115e4c6860 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.fs: Posix File System 2024-09-13T22:23:43.707 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.info_log: 0x56115f6ebfa0 2024-09-13T22:23:43.708 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:23:43.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.statistics: (nil) 2024-09-13T22:23:43.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.use_fsync: 0 2024-09-13T22:23:43.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:23:43.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:23:43.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:23:43.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:23:43.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:23:43.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:23:43.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.db_log_dir: 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi118/store.db 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:23:43.711 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.write_buffer_manager: 0x56115f7dc2d0 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:23:43.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.unordered_write: 0 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.row_cache: None 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.wal_filter: None 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.preserve_deletes: 0 2024-09-13T22:23:43.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:23:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:23:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_open_files: -1 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Compression algorithms supported: 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: kZSTD supported: 0 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:23:43.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi118/store.db/MANIFEST-000003 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.merge_operator: 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_filter: None 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:23:43.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56115f6b7d00) 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: index_type: 0 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: data_block_index_type: 0 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: index_shortening: 1 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: hash_index_allow_collision: 1 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: checksum: 1 2024-09-13T22:23:43.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: no_block_cache: 0 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache: 0x56115f722170 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache_options: 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: capacity : 536870912 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: num_shard_bits : 4 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: strict_capacity_limit : 0 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache_compressed: (nil) 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: persistent_cache: (nil) 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_size: 4096 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_size_deviation: 10 2024-09-13T22:23:43.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_restart_interval: 16 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: index_block_restart_interval: 1 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: metadata_block_size: 4096 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: partition_filters: 0 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: use_delta_encoding: 1 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: whole_key_filtering: 1 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: verify_compression: 0 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: format_version: 4 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: enable_index_compression: 1 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_align: 0 2024-09-13T22:23:43.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compression: NoCompression 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.num_levels: 7 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:23:43.721 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:23:43.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:23:43.723 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:23:43.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.table_properties_collectors: 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:23:43.725 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.ttl: 2592000 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:23:43.726 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi118/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266223458842, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266223459165, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726266223, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "70d48f60-5ce1-4eae-9317-ee673369d9bb", "db_session_id": "2V2HIDCI6W3KVDN1JLKC"}} 2024-09-13T22:23:43.727 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266223459485, "job": 1, "event": "recovery_finished"} 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi118/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x56115f708a80 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: DB pointer 0x56115f718000 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** DB Stats ** 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:23:43.728 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** Compaction Stats [default] ** 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:23:43.729 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** Compaction Stats [default] ** 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:23:43.730 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** Compaction Stats [default] ** 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:23:43.731 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** Compaction Stats [default] ** 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:23:43.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118 does not exist in monmap, will attempt to join an existing cluster 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: using public_addr v2:172.21.15.118:0/0 -> [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: starting mon.smithi118 rank -1 at public addrs [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] at bind addrs [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi118 fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(???) e0 preinit fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).mds e1 new map 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).mds e1 print_map 2024-09-13T22:23:43.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: e1 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: legacy client fscid: -1 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: No filesystems configured 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T22:23:43.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Deploying daemon crash.smithi105 on smithi105 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.735 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3251333457' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Deploying daemon grafana.smithi105 on smithi105 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3251333457' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.736 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi105:172.21.15.105=smithi105;smithi118:172.21.15.118=smithi118", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Saving service mon spec with placement smithi105:172.21.15.105=smithi105;smithi118:172.21.15.118=smithi118;count:2 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/1030761713' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/3997076632' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Deploying daemon node-exporter.smithi105 on smithi105 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/233612315' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.737 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14162 172.21.15.105:0/1345441711' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mgrmap e14: smithi105.swmmea(active, since 43s) 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/1916087297' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/473855600' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Active manager daemon smithi105.swmmea restarted 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Activating manager daemon smithi105.swmmea 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mgrmap e15: smithi105.swmmea(active, starting, since 0.769983s) 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:23:43.738 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Manager daemon smithi105.swmmea is now available 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.739 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: [13/Sep/2024:22:23:26] ENGINE Bus STARTING 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: [13/Sep/2024:22:23:26] ENGINE Serving on https://172.21.15.105:7150 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: [13/Sep/2024:22:23:26] ENGINE Bus STARTED 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mgrmap e16: smithi105.swmmea(active, since 1.77429s) 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mgrmap e17: smithi105.swmmea(active, since 2s) 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/3676715197' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd/host:smithi105", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:23:43.740 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd/host:smithi118", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Deploying daemon crash.smithi118 on smithi118 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/751919740' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.741 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Deploying daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/2460823911' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: Deploying daemon mon.smithi118 on smithi118 2024-09-13T22:23:43.742 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:43.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/2652514894' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:43.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-13T22:23:43.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 ceph-mon[25717]: mon.smithi118@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-13T22:23:43.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 bash[25668]: 17b16ac8afb2e81435a3ccd15b75552a8f5e1c5367f18162469ff95bd6280c01 2024-09-13T22:23:43.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:43 smithi118 systemd[1]: Started Ceph mon.smithi118 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: mon.smithi105 calling monitor election 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/3120155235' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: mon.smithi118 calling monitor election 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: mon.smithi105 is new leader, mons smithi105,smithi118 in quorum (ranks 0,1) 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: monmap e2: 2 mons at {smithi105=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],smithi118=[v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0]} 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: fsmap 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: mgrmap e17: smithi105.swmmea(active, since 23s) 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/3120155235' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: overall HEALTH_OK 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/3120155235' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:48.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:48 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/3120155235' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:23:48.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: mon.smithi105 calling monitor election 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/3120155235' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: mon.smithi118 calling monitor election 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:48.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: mon.smithi105 is new leader, mons smithi105,smithi118 in quorum (ranks 0,1) 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: monmap e2: 2 mons at {smithi105=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],smithi118=[v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0]} 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: fsmap 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: mgrmap e17: smithi105.swmmea(active, since 23s) 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/3120155235' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: overall HEALTH_OK 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/3120155235' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:48.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:48 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/3120155235' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:23:49.043 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:23:49.043 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":2,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","modified":"2024-09-13T22:23:43.483561Z","created":"2024-09-13T22:21:27.083377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi105","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi118","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:3300","nonce":0},{"type":"v1","addr":"172.21.15.118:6789","nonce":0}]},"addr":"172.21.15.118:6789/0","public_addr":"172.21.15.118:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T22:23:49.045 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 2 2024-09-13T22:23:49.644 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:49 smithi118 ceph-mon[25717]: Deploying daemon node-exporter.smithi118 on smithi118 2024-09-13T22:23:49.644 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:49 smithi118 ceph-mon[25717]: mgrmap e18: smithi105.swmmea(active, since 23s), standbys: smithi118.rbhtgz 2024-09-13T22:23:49.644 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:49 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rbhtgz", "id": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:23:49.644 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:49 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/2504779545' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:49.644 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:49 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:49.666 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T22:23:49.666 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T22:23:49.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:49 smithi105 ceph-mon[23707]: Deploying daemon node-exporter.smithi118 on smithi118 2024-09-13T22:23:49.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:49 smithi105 ceph-mon[23707]: mgrmap e18: smithi105.swmmea(active, since 23s), standbys: smithi118.rbhtgz 2024-09-13T22:23:49.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:49 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rbhtgz", "id": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:23:49.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:49 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/2504779545' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T22:23:49.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:49 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:23:50.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:50 smithi118 ceph-mon[25717]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:50.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:50 smithi105 ceph-mon[23707]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:51.546 INFO:teuthology.orchestra.run.smithi105.stdout:# minimal ceph.conf for 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:23:51.546 INFO:teuthology.orchestra.run.smithi105.stdout:[global] 2024-09-13T22:23:51.546 INFO:teuthology.orchestra.run.smithi105.stdout: fsid = 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:23:51.546 INFO:teuthology.orchestra.run.smithi105.stdout: mon_host = [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] 2024-09-13T22:23:51.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:51 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:51.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:51 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2954630944' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:51.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:51 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:51.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:51 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2954630944' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:23:52.251 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T22:23:52.251 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:23:52.251 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T22:23:52.281 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:23:52.281 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:23:52.348 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-13T22:23:52.348 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T22:23:52.379 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-13T22:23:52.379 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:23:52.445 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T22:23:52.445 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:23:52.446 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T22:23:52.461 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T22:23:52.462 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_1 2024-09-13T22:23:52.517 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T22:23:52.517 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T22:23:52.517 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-13T22:23:52.518 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T22:23:52.518 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T22:23:52.518 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-13 22:23:15.283031955 +0000 2024-09-13T22:23:52.518 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-13 22:20:44.944748250 +0000 2024-09-13T22:23:52.518 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-13 22:20:44.944748250 +0000 2024-09-13T22:23:52.518 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-13 22:20:44.944748250 +0000 2024-09-13T22:23:52.518 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T22:23:52.581 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-13T22:23:52.581 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-13T22:23:52.581 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000214596 s, 2.4 MB/s 2024-09-13T22:23:52.582 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T22:23:52.639 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_2 2024-09-13T22:23:52.691 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:52 smithi118 ceph-mon[25717]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:52.694 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T22:23:52.694 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T22:23:52.695 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-13T22:23:52.695 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T22:23:52.695 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T22:23:52.695 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-13 22:23:15.283031955 +0000 2024-09-13T22:23:52.695 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-13 22:20:45.281741250 +0000 2024-09-13T22:23:52.695 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-13 22:20:45.281741250 +0000 2024-09-13T22:23:52.695 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-13 22:20:45.281741250 +0000 2024-09-13T22:23:52.695 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T22:23:52.758 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-13T22:23:52.758 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-13T22:23:52.758 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000240345 s, 2.1 MB/s 2024-09-13T22:23:52.759 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T22:23:52.815 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_3 2024-09-13T22:23:52.871 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T22:23:52.871 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T22:23:52.872 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-13T22:23:52.872 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T22:23:52.872 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T22:23:52.872 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-13 22:23:15.283031955 +0000 2024-09-13T22:23:52.872 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-13 22:20:45.603734562 +0000 2024-09-13T22:23:52.872 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-13 22:20:45.603734562 +0000 2024-09-13T22:23:52.872 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-13 22:20:45.603734562 +0000 2024-09-13T22:23:52.872 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T22:23:52.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:52 smithi105 ceph-mon[23707]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:52.912 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-13T22:23:52.912 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-13T22:23:52.912 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000187874 s, 2.7 MB/s 2024-09-13T22:23:52.914 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T22:23:52.971 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_4 2024-09-13T22:23:53.025 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T22:23:53.026 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T22:23:53.026 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-13T22:23:53.026 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T22:23:53.026 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T22:23:53.026 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-13 22:23:15.284031935 +0000 2024-09-13T22:23:53.026 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-13 22:20:45.915728082 +0000 2024-09-13T22:23:53.026 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-13 22:20:45.915728082 +0000 2024-09-13T22:23:53.026 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-13 22:20:45.915728082 +0000 2024-09-13T22:23:53.027 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T22:23:53.091 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-13T22:23:53.091 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-13T22:23:53.091 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000184298 s, 2.8 MB/s 2024-09-13T22:23:53.092 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T22:23:53.149 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-13T22:23:53.149 DEBUG:teuthology.orchestra.run.smithi118:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T22:23:53.167 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T22:23:53.167 DEBUG:teuthology.orchestra.run.smithi118:> stat /dev/vg_nvme/lv_1 2024-09-13T22:23:53.223 INFO:teuthology.orchestra.run.smithi118.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T22:23:53.223 INFO:teuthology.orchestra.run.smithi118.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T22:23:53.223 INFO:teuthology.orchestra.run.smithi118.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-13T22:23:53.223 INFO:teuthology.orchestra.run.smithi118.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T22:23:53.224 INFO:teuthology.orchestra.run.smithi118.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T22:23:53.224 INFO:teuthology.orchestra.run.smithi118.stdout:Access: 2024-09-13 22:23:49.080840317 +0000 2024-09-13T22:23:53.224 INFO:teuthology.orchestra.run.smithi118.stdout:Modify: 2024-09-13 22:20:46.470068312 +0000 2024-09-13T22:23:53.224 INFO:teuthology.orchestra.run.smithi118.stdout:Change: 2024-09-13 22:20:46.470068312 +0000 2024-09-13T22:23:53.224 INFO:teuthology.orchestra.run.smithi118.stdout: Birth: 2024-09-13 22:20:46.470068312 +0000 2024-09-13T22:23:53.224 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T22:23:53.289 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records in 2024-09-13T22:23:53.289 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records out 2024-09-13T22:23:53.289 INFO:teuthology.orchestra.run.smithi118.stderr:512 bytes copied, 0.000100077 s, 5.1 MB/s 2024-09-13T22:23:53.290 DEBUG:teuthology.orchestra.run.smithi118:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T22:23:53.348 DEBUG:teuthology.orchestra.run.smithi118:> stat /dev/vg_nvme/lv_2 2024-09-13T22:23:53.404 INFO:teuthology.orchestra.run.smithi118.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T22:23:53.404 INFO:teuthology.orchestra.run.smithi118.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T22:23:53.404 INFO:teuthology.orchestra.run.smithi118.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-13T22:23:53.404 INFO:teuthology.orchestra.run.smithi118.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T22:23:53.404 INFO:teuthology.orchestra.run.smithi118.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T22:23:53.404 INFO:teuthology.orchestra.run.smithi118.stdout:Access: 2024-09-13 22:23:49.080840317 +0000 2024-09-13T22:23:53.404 INFO:teuthology.orchestra.run.smithi118.stdout:Modify: 2024-09-13 22:20:46.783066769 +0000 2024-09-13T22:23:53.404 INFO:teuthology.orchestra.run.smithi118.stdout:Change: 2024-09-13 22:20:46.783066769 +0000 2024-09-13T22:23:53.405 INFO:teuthology.orchestra.run.smithi118.stdout: Birth: 2024-09-13 22:20:46.783066769 +0000 2024-09-13T22:23:53.405 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T22:23:53.468 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records in 2024-09-13T22:23:53.468 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records out 2024-09-13T22:23:53.468 INFO:teuthology.orchestra.run.smithi118.stderr:512 bytes copied, 0.000108711 s, 4.7 MB/s 2024-09-13T22:23:53.469 DEBUG:teuthology.orchestra.run.smithi118:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T22:23:53.526 DEBUG:teuthology.orchestra.run.smithi118:> stat /dev/vg_nvme/lv_3 2024-09-13T22:23:53.582 INFO:teuthology.orchestra.run.smithi118.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T22:23:53.582 INFO:teuthology.orchestra.run.smithi118.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T22:23:53.582 INFO:teuthology.orchestra.run.smithi118.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-13T22:23:53.582 INFO:teuthology.orchestra.run.smithi118.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T22:23:53.582 INFO:teuthology.orchestra.run.smithi118.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T22:23:53.582 INFO:teuthology.orchestra.run.smithi118.stdout:Access: 2024-09-13 22:23:49.081840312 +0000 2024-09-13T22:23:53.582 INFO:teuthology.orchestra.run.smithi118.stdout:Modify: 2024-09-13 22:20:47.100065206 +0000 2024-09-13T22:23:53.582 INFO:teuthology.orchestra.run.smithi118.stdout:Change: 2024-09-13 22:20:47.100065206 +0000 2024-09-13T22:23:53.582 INFO:teuthology.orchestra.run.smithi118.stdout: Birth: 2024-09-13 22:20:47.100065206 +0000 2024-09-13T22:23:53.583 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T22:23:53.645 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records in 2024-09-13T22:23:53.645 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records out 2024-09-13T22:23:53.645 INFO:teuthology.orchestra.run.smithi118.stderr:512 bytes copied, 0.000130392 s, 3.9 MB/s 2024-09-13T22:23:53.646 DEBUG:teuthology.orchestra.run.smithi118:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T22:23:53.703 DEBUG:teuthology.orchestra.run.smithi118:> stat /dev/vg_nvme/lv_4 2024-09-13T22:23:53.758 INFO:teuthology.orchestra.run.smithi118.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T22:23:53.758 INFO:teuthology.orchestra.run.smithi118.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T22:23:53.758 INFO:teuthology.orchestra.run.smithi118.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-13T22:23:53.758 INFO:teuthology.orchestra.run.smithi118.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T22:23:53.758 INFO:teuthology.orchestra.run.smithi118.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T22:23:53.758 INFO:teuthology.orchestra.run.smithi118.stdout:Access: 2024-09-13 22:23:49.081840312 +0000 2024-09-13T22:23:53.758 INFO:teuthology.orchestra.run.smithi118.stdout:Modify: 2024-09-13 22:20:47.400063727 +0000 2024-09-13T22:23:53.758 INFO:teuthology.orchestra.run.smithi118.stdout:Change: 2024-09-13 22:20:47.400063727 +0000 2024-09-13T22:23:53.759 INFO:teuthology.orchestra.run.smithi118.stdout: Birth: 2024-09-13 22:20:47.400063727 +0000 2024-09-13T22:23:53.759 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T22:23:53.821 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records in 2024-09-13T22:23:53.821 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records out 2024-09-13T22:23:53.821 INFO:teuthology.orchestra.run.smithi118.stderr:512 bytes copied, 9.0389e-05 s, 5.7 MB/s 2024-09-13T22:23:53.822 DEBUG:teuthology.orchestra.run.smithi118:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T22:23:53.879 INFO:tasks.cephadm:Deploying osd.0 on smithi105 with /dev/vg_nvme/lv_4... 2024-09-13T22:23:53.879 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T22:23:54.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:53 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:54.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:53 smithi118 ceph-mon[25717]: Deploying daemon prometheus.smithi105 on smithi105 2024-09-13T22:23:54.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:53 smithi118 ceph-mon[25717]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:54.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:53 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:54.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:53 smithi105 ceph-mon[23707]: Deploying daemon prometheus.smithi105 on smithi105 2024-09-13T22:23:54.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:53 smithi105 ceph-mon[23707]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:56.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:56 smithi118 ceph-mon[25717]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:56.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:56 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:56.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:56 smithi105 ceph-mon[23707]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:56.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:56 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:23:58.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:23:58 smithi105 ceph-mon[23707]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:23:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:23:58 smithi118 ceph-mon[25717]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:00.732 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:00 smithi105 ceph-mon[23707]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:00 smithi118 ceph-mon[25717]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:02.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:02 smithi105 ceph-mon[23707]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:02 smithi118 ceph-mon[25717]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:04.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:04 smithi105 ceph-mon[23707]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:04.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:04.549 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:24:04.549 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:04.549 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:24:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:04 smithi118 ceph-mon[25717]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:24:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:24:05.651 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:24:05.670 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_4 2024-09-13T22:24:06.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:06 smithi105 ceph-mon[23707]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:06.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:06 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:06.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:06 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:06.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:06 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:06 smithi118 ceph-mon[25717]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:06 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:06 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:06 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:07.800 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:07 smithi105 ceph-mon[23707]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-13T22:24:07.800 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:07 smithi105 ceph-mon[23707]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:24:08.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:07 smithi118 ceph-mon[25717]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-13T22:24:08.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:07 smithi118 ceph-mon[25717]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:24:08.713 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:08 smithi105 ceph-mon[23707]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:08.713 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:08 smithi105 ceph-mon[23707]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:24:08.713 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:08 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:24:08.713 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:08 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:24:08.713 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:08 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:08 smithi118 ceph-mon[25717]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:08 smithi118 ceph-mon[25717]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:24:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:08 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:24:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:08 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:24:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:08 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:10 smithi105 ceph-mon[23707]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:10 smithi118 ceph-mon[25717]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:11.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:11 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3264062289' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8b91105-6278-4f62-8a38-6a9b2964f21b"}]: dispatch 2024-09-13T22:24:11.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:11 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3264062289' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8b91105-6278-4f62-8a38-6a9b2964f21b"}]': finished 2024-09-13T22:24:11.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:11 smithi105 ceph-mon[23707]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T22:24:11.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:11 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:11 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3264062289' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8b91105-6278-4f62-8a38-6a9b2964f21b"}]: dispatch 2024-09-13T22:24:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:11 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3264062289' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8b91105-6278-4f62-8a38-6a9b2964f21b"}]': finished 2024-09-13T22:24:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:11 smithi118 ceph-mon[25717]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T22:24:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:11 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:12 smithi118 ceph-mon[25717]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:12 smithi118 ceph-mon[25717]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:24:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:13.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:24:13.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:24:13.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:13.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:12 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/1988479321' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:24:13.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:12 smithi105 ceph-mon[23707]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:13.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:13.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:12 smithi105 ceph-mon[23707]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:24:13.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:13.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:13.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:24:13.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:24:13.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:13.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:12 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1988479321' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:24:14.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:13 smithi105 ceph-mon[23707]: Reconfiguring mon.smithi105 (unknown last config time)... 2024-09-13T22:24:14.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:13 smithi105 ceph-mon[23707]: Reconfiguring daemon mon.smithi105 on smithi105 2024-09-13T22:24:14.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:13 smithi118 ceph-mon[25717]: Reconfiguring mon.smithi105 (unknown last config time)... 2024-09-13T22:24:14.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:13 smithi118 ceph-mon[25717]: Reconfiguring daemon mon.smithi105 on smithi105 2024-09-13T22:24:15.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:14 smithi105 ceph-mon[23707]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:15.248 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:14 smithi118 ceph-mon[25717]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:16.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:16 smithi105 ceph-mon[23707]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:16.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:16.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi105.swmmea", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:24:16.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:24:16.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:17.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:16 smithi118 ceph-mon[25717]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:17.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:17.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi105.swmmea", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:24:17.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:24:17.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:18.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:17 smithi105 ceph-mon[23707]: Reconfiguring mgr.smithi105.swmmea (unknown last config time)... 2024-09-13T22:24:18.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:17 smithi105 ceph-mon[23707]: Reconfiguring daemon mgr.smithi105.swmmea on smithi105 2024-09-13T22:24:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:17 smithi118 ceph-mon[25717]: Reconfiguring mgr.smithi105.swmmea (unknown last config time)... 2024-09-13T22:24:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:17 smithi118 ceph-mon[25717]: Reconfiguring daemon mgr.smithi105.swmmea on smithi105 2024-09-13T22:24:19.113 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:18 smithi105 ceph-mon[23707]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:19.113 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:18 smithi118 ceph-mon[25717]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:20.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:19 smithi105 ceph-mon[23707]: Reconfiguring alertmanager.smithi105 (dependencies changed)... 2024-09-13T22:24:20.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:19 smithi105 ceph-mon[23707]: Reconfiguring daemon alertmanager.smithi105 on smithi105 2024-09-13T22:24:20.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:19 smithi118 ceph-mon[25717]: Reconfiguring alertmanager.smithi105 (dependencies changed)... 2024-09-13T22:24:20.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:19 smithi118 ceph-mon[25717]: Reconfiguring daemon alertmanager.smithi105 on smithi105 2024-09-13T22:24:21.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:20 smithi105 ceph-mon[23707]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:21.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:24:21.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:21.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:20 smithi118 ceph-mon[25717]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:21.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:24:21.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:22.129 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:21 smithi105 ceph-mon[23707]: Deploying daemon osd.0 on smithi105 2024-09-13T22:24:22.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:21 smithi118 ceph-mon[25717]: Deploying daemon osd.0 on smithi105 2024-09-13T22:24:23.004 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:22 smithi105 ceph-mon[23707]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:23.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:22 smithi118 ceph-mon[25717]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:24.387 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:24 smithi105 ceph-mon[23707]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:24.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:24 smithi118 ceph-mon[25717]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:26.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:26.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:25 smithi105 ceph-mon[23707]: Reconfiguring crash.smithi105 (monmap changed)... 2024-09-13T22:24:26.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi105", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:24:26.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:26.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:25 smithi105 ceph-mon[23707]: Reconfiguring daemon crash.smithi105 on smithi105 2024-09-13T22:24:26.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:25 smithi105 ceph-mon[23707]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:26.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:24:26.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:24:26.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:26.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:25 smithi118 ceph-mon[25717]: Reconfiguring crash.smithi105 (monmap changed)... 2024-09-13T22:24:26.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi105", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:24:26.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:26.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:25 smithi118 ceph-mon[25717]: Reconfiguring daemon crash.smithi105 on smithi105 2024-09-13T22:24:26.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:25 smithi118 ceph-mon[25717]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:26.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:24:26.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:24:28.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:28 smithi105 ceph-mon[23707]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:28 smithi118 ceph-mon[25717]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:30.698 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:30 smithi105 ceph-mon[23707]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:30 smithi118 ceph-mon[25717]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:32.250 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:32.250 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:32 smithi105 ceph-mon[23707]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:32.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:32 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:32.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:32 smithi118 ceph-mon[25717]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:34.149 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:33 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:34.150 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:33 smithi105 ceph-mon[23707]: Reconfiguring grafana.smithi105 (dependencies changed)... 2024-09-13T22:24:34.150 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:33 smithi105 ceph-mon[23707]: Reconfiguring daemon grafana.smithi105 on smithi105 2024-09-13T22:24:34.150 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:33 smithi105 ceph-mon[23707]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:33 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:33 smithi118 ceph-mon[25717]: Reconfiguring grafana.smithi105 (dependencies changed)... 2024-09-13T22:24:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:33 smithi118 ceph-mon[25717]: Reconfiguring daemon grafana.smithi105 on smithi105 2024-09-13T22:24:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:33 smithi118 ceph-mon[25717]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:34.942 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 0 on host 'smithi105' 2024-09-13T22:24:35.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:34 smithi105 ceph-mon[23707]: from='osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:24:35.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:34 smithi118 ceph-mon[25717]: from='osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:24:35.555 DEBUG:teuthology.orchestra.run.smithi105:osd.0> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T22:24:35.557 INFO:tasks.cephadm:Deploying osd.1 on smithi105 with /dev/vg_nvme/lv_3... 2024-09-13T22:24:35.557 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T22:24:36.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:35 smithi105 ceph-mon[23707]: from='osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:24:36.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:35 smithi105 ceph-mon[23707]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T22:24:36.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:35 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:36.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:35 smithi105 ceph-mon[23707]: from='osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:24:36.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:35 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:36.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:35 smithi105 ceph-mon[23707]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:36.141 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:24:35 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[34217]: 2024-09-13T22:24:35.906+0000 7f9b124a1700 -1 osd.0 0 waiting for initial osdmap 2024-09-13T22:24:36.141 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:24:35 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[34217]: 2024-09-13T22:24:35.909+0000 7f9b0ce38700 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:24:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:35 smithi118 ceph-mon[25717]: from='osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:24:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:35 smithi118 ceph-mon[25717]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T22:24:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:35 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:35 smithi118 ceph-mon[25717]: from='osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:24:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:35 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:35 smithi118 ceph-mon[25717]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:37.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:36 smithi105 ceph-mon[23707]: from='osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-13T22:24:37.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:36 smithi105 ceph-mon[23707]: osdmap e8: 1 total, 0 up, 1 in 2024-09-13T22:24:37.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:36 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:37.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:36 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:37.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:36 smithi118 ceph-mon[25717]: from='osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-13T22:24:37.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:36 smithi118 ceph-mon[25717]: osdmap e8: 1 total, 0 up, 1 in 2024-09-13T22:24:37.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:36 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:37.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:36 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:38.110 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:37 smithi105 ceph-mon[23707]: purged_snaps scrub starts 2024-09-13T22:24:38.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:37 smithi105 ceph-mon[23707]: purged_snaps scrub ok 2024-09-13T22:24:38.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:37 smithi105 ceph-mon[23707]: osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744] boot 2024-09-13T22:24:38.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:37 smithi105 ceph-mon[23707]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T22:24:38.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:37 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:38.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:37 smithi105 ceph-mon[23707]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:37 smithi118 ceph-mon[25717]: purged_snaps scrub starts 2024-09-13T22:24:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:37 smithi118 ceph-mon[25717]: purged_snaps scrub ok 2024-09-13T22:24:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:37 smithi118 ceph-mon[25717]: osd.0 [v2:172.21.15.105:6802/770182744,v1:172.21.15.105:6803/770182744] boot 2024-09-13T22:24:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:37 smithi118 ceph-mon[25717]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T22:24:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:37 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:24:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:37 smithi118 ceph-mon[25717]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T22:24:39.050 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:38 smithi105 ceph-mon[23707]: osdmap e10: 1 total, 1 up, 1 in 2024-09-13T22:24:39.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:38 smithi118 ceph-mon[25717]: osdmap e10: 1 total, 1 up, 1 in 2024-09-13T22:24:40.182 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:39 smithi105 ceph-mon[23707]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:39 smithi118 ceph-mon[25717]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:41.634 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:41 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:41.634 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:41 smithi105 ceph-mon[23707]: Reconfiguring crash.smithi118 (monmap changed)... 2024-09-13T22:24:41.634 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:41 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:24:41.634 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:41 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:41.635 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:41 smithi105 ceph-mon[23707]: Reconfiguring daemon crash.smithi118 on smithi118 2024-09-13T22:24:41.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:41 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:41.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:41 smithi118 ceph-mon[25717]: Reconfiguring crash.smithi118 (monmap changed)... 2024-09-13T22:24:41.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:41 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:24:41.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:41 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:41.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:41 smithi118 ceph-mon[25717]: Reconfiguring daemon crash.smithi118 on smithi118 2024-09-13T22:24:41.923 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:24:41.942 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_3 2024-09-13T22:24:42.783 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:42 smithi118 ceph-mon[25717]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:42.784 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:42.784 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:24:42.784 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:24:42.784 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:42 smithi105 ceph-mon[23707]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:24:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:24:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:43 smithi118 ceph-mon[25717]: Reconfiguring mgr.smithi118.rbhtgz (monmap changed)... 2024-09-13T22:24:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:43 smithi118 ceph-mon[25717]: Reconfiguring daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:24:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:24:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:24:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:43.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:43 smithi105 ceph-mon[23707]: Reconfiguring mgr.smithi118.rbhtgz (monmap changed)... 2024-09-13T22:24:43.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:43 smithi105 ceph-mon[23707]: Reconfiguring daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:24:43.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:43 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:43.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:43 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:24:43.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:43 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:24:43.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:43 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:44 smithi118 ceph-mon[25717]: Reconfiguring mon.smithi118 (monmap changed)... 2024-09-13T22:24:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:44 smithi118 ceph-mon[25717]: Reconfiguring daemon mon.smithi118 on smithi118 2024-09-13T22:24:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:44 smithi118 ceph-mon[25717]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:44 smithi118 ceph-mon[25717]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:24:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:44 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:24:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:44 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:24:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:44 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:44.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:44 smithi105 ceph-mon[23707]: Reconfiguring mon.smithi118 (monmap changed)... 2024-09-13T22:24:44.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:44 smithi105 ceph-mon[23707]: Reconfiguring daemon mon.smithi118 on smithi118 2024-09-13T22:24:44.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:44 smithi105 ceph-mon[23707]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:44.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:44 smithi105 ceph-mon[23707]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:24:44.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:44 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:24:44.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:44 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:24:44.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:44 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:45.793 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:45 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:45.793 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:45 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:24:45.793 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:45 smithi118 ceph-mon[25717]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:24:45.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:45 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:45.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:45 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:24:45.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:45 smithi105 ceph-mon[23707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:24:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:46 smithi118 ceph-mon[25717]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:46 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2346624' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "49d94541-03ae-40a1-9fcd-abd7730bc069"}]: dispatch 2024-09-13T22:24:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:46 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2346624' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "49d94541-03ae-40a1-9fcd-abd7730bc069"}]': finished 2024-09-13T22:24:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:46 smithi118 ceph-mon[25717]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T22:24:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:46 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:24:46.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:46 smithi105 ceph-mon[23707]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:46.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:46 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2346624' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "49d94541-03ae-40a1-9fcd-abd7730bc069"}]: dispatch 2024-09-13T22:24:46.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:46 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2346624' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "49d94541-03ae-40a1-9fcd-abd7730bc069"}]': finished 2024-09-13T22:24:46.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:46 smithi105 ceph-mon[23707]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T22:24:46.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:46 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:24:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:47 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3891231413' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:24:47.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:47 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3891231413' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:24:48.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:48 smithi118 ceph-mon[25717]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:48.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:48 smithi105 ceph-mon[23707]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:50.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:50 smithi105 ceph-mon[23707]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:50.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:50 smithi118 ceph-mon[25717]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:52.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:52 smithi105 ceph-mon[23707]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:52.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:52 smithi118 ceph-mon[25717]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:53.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:53 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T22:24:53.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:53 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:53.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:53 smithi105 ceph-mon[23707]: Deploying daemon osd.1 on smithi105 2024-09-13T22:24:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:53 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T22:24:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:53 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:24:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:53 smithi118 ceph-mon[25717]: Deploying daemon osd.1 on smithi105 2024-09-13T22:24:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:54 smithi118 ceph-mon[25717]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:54.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:54 smithi105 ceph-mon[23707]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:56.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:56 smithi118 ceph-mon[25717]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:56.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:56 smithi105 ceph-mon[23707]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:59.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:58 smithi105 ceph-mon[23707]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:59.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:24:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:24:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:58 smithi118 ceph-mon[25717]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:24:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:24:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:00.646 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 1 on host 'smithi105' 2024-09-13T22:25:00.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:25:00.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.105:9093"}]: dispatch 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:00.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:25:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.105:9093"}]: dispatch 2024-09-13T22:25:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:25:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-13T22:25:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:25:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-13T22:25:01.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:01.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:01.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:01.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:01.447 DEBUG:teuthology.orchestra.run.smithi105:osd.1> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T22:25:01.449 INFO:tasks.cephadm:Deploying osd.2 on smithi105 with /dev/vg_nvme/lv_2... 2024-09-13T22:25:01.449 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T22:25:02.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:01 smithi105 ceph-mon[23707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.105:9093"}]: dispatch 2024-09-13T22:25:02.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:01 smithi105 ceph-mon[23707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:25:02.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:01 smithi105 ceph-mon[23707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-13T22:25:02.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:01 smithi105 ceph-mon[23707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:25:02.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:01 smithi105 ceph-mon[23707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-13T22:25:02.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:01 smithi105 ceph-mon[23707]: from='osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T22:25:02.312 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:01 smithi118 ceph-mon[25717]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.105:9093"}]: dispatch 2024-09-13T22:25:02.313 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:01 smithi118 ceph-mon[25717]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:25:02.313 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:01 smithi118 ceph-mon[25717]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.105:3000"}]: dispatch 2024-09-13T22:25:02.313 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:01 smithi118 ceph-mon[25717]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:25:02.313 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:01 smithi118 ceph-mon[25717]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.105:9095"}]: dispatch 2024-09-13T22:25:02.313 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:01 smithi118 ceph-mon[25717]: from='osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T22:25:03.089 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:02 smithi105 ceph-mon[23707]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:25:03.090 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:02 smithi105 ceph-mon[23707]: from='osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T22:25:03.090 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:02 smithi105 ceph-mon[23707]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T22:25:03.090 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:02 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:25:03.090 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:02 smithi105 ceph-mon[23707]: from='osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:25:03.090 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:02 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:03.090 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:25:02 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[37058]: 2024-09-13T22:25:02.862+0000 7fb0256b7700 -1 osd.1 0 waiting for initial osdmap 2024-09-13T22:25:03.090 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:25:02 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[37058]: 2024-09-13T22:25:02.865+0000 7fb022052700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:25:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:02 smithi118 ceph-mon[25717]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:25:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:02 smithi118 ceph-mon[25717]: from='osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T22:25:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:02 smithi118 ceph-mon[25717]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T22:25:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:02 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:25:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:02 smithi118 ceph-mon[25717]: from='osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:25:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:02 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:04.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:03 smithi105 ceph-mon[23707]: from='osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-13T22:25:04.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:03 smithi105 ceph-mon[23707]: osdmap e13: 2 total, 1 up, 2 in 2024-09-13T22:25:04.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:03 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:25:04.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:03 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:25:04.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:03 smithi118 ceph-mon[25717]: from='osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-13T22:25:04.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:03 smithi118 ceph-mon[25717]: osdmap e13: 2 total, 1 up, 2 in 2024-09-13T22:25:04.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:03 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:25:04.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:03 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:25:04.961 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:04 smithi105 ceph-mon[23707]: purged_snaps scrub starts 2024-09-13T22:25:04.961 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:04 smithi105 ceph-mon[23707]: purged_snaps scrub ok 2024-09-13T22:25:04.961 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:04 smithi105 ceph-mon[23707]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:25:04.961 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:04 smithi105 ceph-mon[23707]: osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047] boot 2024-09-13T22:25:04.961 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:04 smithi105 ceph-mon[23707]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T22:25:04.961 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:25:05.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:04 smithi118 ceph-mon[25717]: purged_snaps scrub starts 2024-09-13T22:25:05.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:04 smithi118 ceph-mon[25717]: purged_snaps scrub ok 2024-09-13T22:25:05.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:04 smithi118 ceph-mon[25717]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T22:25:05.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:04 smithi118 ceph-mon[25717]: osd.1 [v2:172.21.15.105:6810/996806047,v1:172.21.15.105:6811/996806047] boot 2024-09-13T22:25:05.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:04 smithi118 ceph-mon[25717]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T22:25:05.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:25:05.657 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:25:05.677 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_2 2024-09-13T22:25:05.980 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:05 smithi105 ceph-mon[23707]: osdmap e15: 2 total, 2 up, 2 in 2024-09-13T22:25:05.981 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:05 smithi105 ceph-mon[23707]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:06.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:05 smithi118 ceph-mon[25717]: osdmap e15: 2 total, 2 up, 2 in 2024-09-13T22:25:06.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:05 smithi118 ceph-mon[25717]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:09.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:08 smithi105 ceph-mon[23707]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:09.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:08 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:08 smithi118 ceph-mon[25717]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:08 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:09.959 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:09 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:25:09.960 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:09 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:25:09.960 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:09 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:09 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:25:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:09 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:25:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:09 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:10.859 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:10 smithi105 ceph-mon[23707]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:25:10.859 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:10 smithi105 ceph-mon[23707]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:10 smithi118 ceph-mon[25717]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:25:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:10 smithi118 ceph-mon[25717]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:12 smithi118 ceph-mon[25717]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:12.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:12.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:12.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:12.912 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:12 smithi105 ceph-mon[23707]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:12.913 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:12.913 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:12.913 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:12.913 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:12.913 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:12.913 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:12.913 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:12.913 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:13 smithi105 ceph-mon[23707]: Detected new or changed devices on smithi105 2024-09-13T22:25:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:13 smithi105 ceph-mon[23707]: Adjusting osd_memory_target on smithi105 to 6432M 2024-09-13T22:25:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:13 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2456829292' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "123dd0dc-4fd0-4620-8f36-6e63822846ed"}]: dispatch 2024-09-13T22:25:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:13 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2456829292' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "123dd0dc-4fd0-4620-8f36-6e63822846ed"}]': finished 2024-09-13T22:25:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:13 smithi105 ceph-mon[23707]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T22:25:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:13 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:14.093 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:13 smithi118 ceph-mon[25717]: Detected new or changed devices on smithi105 2024-09-13T22:25:14.093 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:13 smithi118 ceph-mon[25717]: Adjusting osd_memory_target on smithi105 to 6432M 2024-09-13T22:25:14.093 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:13 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2456829292' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "123dd0dc-4fd0-4620-8f36-6e63822846ed"}]: dispatch 2024-09-13T22:25:14.093 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:13 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2456829292' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "123dd0dc-4fd0-4620-8f36-6e63822846ed"}]': finished 2024-09-13T22:25:14.094 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:13 smithi118 ceph-mon[25717]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T22:25:14.094 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:13 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:14.965 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:14 smithi105 ceph-mon[23707]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:14.965 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:14 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:14.965 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:14 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2278892297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:25:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:14 smithi118 ceph-mon[25717]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:14 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:14 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2278892297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:25:17.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:16 smithi105 ceph-mon[23707]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:16 smithi118 ceph-mon[25717]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:18.775 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:18.776 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:18 smithi105 ceph-mon[23707]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:18.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:18.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:18 smithi118 ceph-mon[25717]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:20.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:20 smithi105 ceph-mon[23707]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:20 smithi118 ceph-mon[25717]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:21.917 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:21 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T22:25:21.917 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:21 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:22.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:21 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T22:25:22.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:21 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:22.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:22 smithi105 ceph-mon[23707]: Deploying daemon osd.2 on smithi105 2024-09-13T22:25:22.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:22 smithi105 ceph-mon[23707]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:23.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:22 smithi118 ceph-mon[25717]: Deploying daemon osd.2 on smithi105 2024-09-13T22:25:23.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:22 smithi118 ceph-mon[25717]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:24.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:24 smithi105 ceph-mon[23707]: Detected new or changed devices on smithi105 2024-09-13T22:25:24.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:24.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:24.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:24 smithi105 ceph-mon[23707]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:24.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:24 smithi118 ceph-mon[25717]: Detected new or changed devices on smithi105 2024-09-13T22:25:24.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:24.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:24.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:24 smithi118 ceph-mon[25717]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:26.089 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:25:26.089 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:25:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:25:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:25:26.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:26 smithi105 ceph-mon[23707]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:27.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:26 smithi118 ceph-mon[25717]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:28.973 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:28 smithi105 ceph-mon[23707]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:28.973 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:28 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:28.973 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:28 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:28.973 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:28 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:28.973 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:28 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:29.000 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:28 smithi118 ceph-mon[25717]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:29.000 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:28 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:29.001 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:28 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:29.001 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:28 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:29.001 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:28 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:30 smithi118 ceph-mon[25717]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:31.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:30 smithi105 ceph-mon[23707]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:31.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:31.740 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:31 smithi105 ceph-mon[23707]: from='osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T22:25:31.868 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 2 on host 'smithi105' 2024-09-13T22:25:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:31 smithi118 ceph-mon[25717]: from='osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T22:25:32.466 DEBUG:teuthology.orchestra.run.smithi105:osd.2> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T22:25:32.469 INFO:tasks.cephadm:Deploying osd.3 on smithi105 with /dev/vg_nvme/lv_1... 2024-09-13T22:25:32.469 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T22:25:32.692 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:25:32 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[40957]: 2024-09-13T22:25:32.679+0000 7f756acd5700 -1 osd.2 0 waiting for initial osdmap 2024-09-13T22:25:32.692 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:25:32 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[40957]: 2024-09-13T22:25:32.682+0000 7f7564e6b700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:25:32.693 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:32 smithi105 ceph-mon[23707]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:32.693 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:32 smithi105 ceph-mon[23707]: from='osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T22:25:32.693 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:32 smithi105 ceph-mon[23707]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T22:25:32.693 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:32.693 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:32 smithi105 ceph-mon[23707]: from='osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:25:32.693 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:32 smithi118 ceph-mon[25717]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:32 smithi118 ceph-mon[25717]: from='osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T22:25:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:32 smithi118 ceph-mon[25717]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T22:25:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:32 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:32 smithi118 ceph-mon[25717]: from='osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:25:33.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:32 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:33.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:33 smithi105 ceph-mon[23707]: from='osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-13T22:25:33.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:33 smithi105 ceph-mon[23707]: osdmap e18: 3 total, 2 up, 3 in 2024-09-13T22:25:33.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:33 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:33.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:33 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:33 smithi118 ceph-mon[25717]: from='osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-13T22:25:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:33 smithi118 ceph-mon[25717]: osdmap e18: 3 total, 2 up, 3 in 2024-09-13T22:25:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:33 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:33 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:34.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:34 smithi105 ceph-mon[23707]: purged_snaps scrub starts 2024-09-13T22:25:34.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:34 smithi105 ceph-mon[23707]: purged_snaps scrub ok 2024-09-13T22:25:34.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:34 smithi105 ceph-mon[23707]: pgmap v70: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:34.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:34 smithi105 ceph-mon[23707]: osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821] boot 2024-09-13T22:25:34.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:34 smithi105 ceph-mon[23707]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T22:25:34.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:34 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:34 smithi118 ceph-mon[25717]: purged_snaps scrub starts 2024-09-13T22:25:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:34 smithi118 ceph-mon[25717]: purged_snaps scrub ok 2024-09-13T22:25:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:34 smithi118 ceph-mon[25717]: pgmap v70: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-13T22:25:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:34 smithi118 ceph-mon[25717]: osd.2 [v2:172.21.15.105:6818/1795261821,v1:172.21.15.105:6819/1795261821] boot 2024-09-13T22:25:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:34 smithi118 ceph-mon[25717]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T22:25:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:34 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:25:36.066 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:25:36.084 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_1 2024-09-13T22:25:36.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:36 smithi118 ceph-mon[25717]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T22:25:36.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:36 smithi118 ceph-mon[25717]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:36.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:36 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T22:25:36.615 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:36 smithi105 ceph-mon[23707]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T22:25:36.615 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:36 smithi105 ceph-mon[23707]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:36.615 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:36 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T22:25:37.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:37 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T22:25:37.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:37 smithi118 ceph-mon[25717]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T22:25:37.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:37 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T22:25:37.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:37 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T22:25:37.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:37 smithi105 ceph-mon[23707]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T22:25:37.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:37 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T22:25:37.956 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:25:37 smithi105 sudo[42277]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T22:25:37.956 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:25:37 smithi105 sudo[42277]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:37.956 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:25:37 smithi105 sudo[42277]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:38.301 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42309]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T22:25:38.301 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42309]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:38.301 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42309]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:38.301 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42317]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T22:25:38.566 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T22:25:38.566 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:38 smithi105 ceph-mon[23707]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T22:25:38.566 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:38 smithi105 ceph-mon[23707]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:38.566 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:38.567 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42317]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:38.567 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42317]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:38.824 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42354]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T22:25:38.824 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42354]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:38.824 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42354]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:38.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T22:25:38.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:38 smithi118 ceph-mon[25717]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T22:25:38.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:38 smithi118 ceph-mon[25717]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:38.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:39.093 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42415]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T22:25:39.094 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42415]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:39.094 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:25:38 smithi105 sudo[42415]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:39.344 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:25:39 smithi105 sudo[42514]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T22:25:39.344 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:25:39 smithi105 sudo[42514]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:39.344 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:25:39 smithi105 sudo[42514]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:39.345 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 sudo[42546]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T22:25:39.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 sudo[42546]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:39.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 sudo[42546]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:39.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:39 smithi118 ceph-mon[25717]: osdmap e23: 3 total, 3 up, 3 in 2024-09-13T22:25:39.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:39 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:39.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:39 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:25:39.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:39 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:25:39.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:39 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:39.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 sudo[42563]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T22:25:39.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 sudo[42563]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:39.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 sudo[42563]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:39.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 ceph-mon[23707]: osdmap e23: 3 total, 3 up, 3 in 2024-09-13T22:25:39.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:39.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:25:39.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:25:39.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:39 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:40.109 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:39 smithi118 sudo[27986]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T22:25:40.109 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:39 smithi118 sudo[27986]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:40.110 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:39 smithi118 sudo[27986]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:40.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 sudo[27989]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T22:25:40.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 sudo[27989]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T22:25:40.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 sudo[27989]: pam_unix(sudo:session): session closed for user root 2024-09-13T22:25:40.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:25:40.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T22:25:40.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:40.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T22:25:40.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:25:40.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:25:40.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T22:25:40.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:25:40.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:25:40.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T22:25:40.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: mgrmap e19: smithi105.swmmea(active, since 2m), standbys: smithi118.rbhtgz 2024-09-13T22:25:40.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:40 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:25:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T22:25:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T22:25:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:25:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:25:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T22:25:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:25:41.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:25:41.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T22:25:41.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: mgrmap e19: smithi105.swmmea(active, since 2m), standbys: smithi118.rbhtgz 2024-09-13T22:25:41.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:40 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:42 smithi105 ceph-mon[23707]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:42.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:42.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:42.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:42 smithi118 ceph-mon[25717]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:25:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:43.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:43 smithi105 ceph-mon[23707]: Adjusting osd_memory_target on smithi105 to 4288M 2024-09-13T22:25:43.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:43 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:43.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:43 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:43.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:43 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:43.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:43 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:44.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:43 smithi118 ceph-mon[25717]: Adjusting osd_memory_target on smithi105 to 4288M 2024-09-13T22:25:44.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:44.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:44.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:44.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:44.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:44 smithi105 ceph-mon[23707]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:44.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:44 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/441532680' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bacf1e21-97ba-4157-8d3f-537c851503cc"}]: dispatch 2024-09-13T22:25:44.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:44 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/441532680' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bacf1e21-97ba-4157-8d3f-537c851503cc"}]': finished 2024-09-13T22:25:44.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:44 smithi105 ceph-mon[23707]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T22:25:44.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:44 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:25:44.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:44 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3498104260' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:25:44.894 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:44 smithi118 ceph-mon[25717]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:44.894 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:44 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/441532680' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bacf1e21-97ba-4157-8d3f-537c851503cc"}]: dispatch 2024-09-13T22:25:44.894 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:44 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/441532680' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bacf1e21-97ba-4157-8d3f-537c851503cc"}]': finished 2024-09-13T22:25:44.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:44 smithi118 ceph-mon[25717]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T22:25:44.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:44 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:25:44.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:44 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3498104260' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:25:46.884 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:46 smithi105 ceph-mon[23707]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-13T22:25:47.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:46 smithi118 ceph-mon[25717]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-13T22:25:48.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:48 smithi118 ceph-mon[25717]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T22:25:48.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:48.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:48 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:48.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:48 smithi105 ceph-mon[23707]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T22:25:48.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:48.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:48 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:50.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:50 smithi105 ceph-mon[23707]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T22:25:50.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:50 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T22:25:50.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:50 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:50 smithi118 ceph-mon[25717]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T22:25:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:50 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T22:25:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:50 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:51.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:51 smithi105 ceph-mon[23707]: Deploying daemon osd.3 on smithi105 2024-09-13T22:25:52.096 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:51 smithi118 ceph-mon[25717]: Deploying daemon osd.3 on smithi105 2024-09-13T22:25:52.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:52 smithi105 ceph-mon[23707]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T22:25:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:52 smithi118 ceph-mon[25717]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T22:25:54.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:54 smithi118 ceph-mon[25717]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T22:25:54.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:54 smithi105 ceph-mon[23707]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T22:25:56.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:56 smithi105 ceph-mon[23707]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2024-09-13T22:25:56.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:56 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:56.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:56 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:56.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:56 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:56.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:56 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:56 smithi118 ceph-mon[25717]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2024-09-13T22:25:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:56 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:56 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:25:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:56 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:25:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:56 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:25:58.726 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:58 smithi105 ceph-mon[23707]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:58.849 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 3 on host 'smithi105' 2024-09-13T22:25:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:58 smithi118 ceph-mon[25717]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:25:59.555 DEBUG:teuthology.orchestra.run.smithi105:osd.3> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T22:25:59.557 INFO:tasks.cephadm:Deploying osd.4 on smithi118 with /dev/vg_nvme/lv_4... 2024-09-13T22:25:59.558 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T22:25:59.785 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:59 smithi105 ceph-mon[23707]: from='osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T22:25:59.785 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:25:59 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:25:59.876 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:59 smithi118 ceph-mon[25717]: from='osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T22:25:59.877 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:25:59 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:00.739 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:00 smithi105 ceph-mon[23707]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:26:00.739 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:00 smithi105 ceph-mon[23707]: from='osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T22:26:00.739 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:00 smithi105 ceph-mon[23707]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T22:26:00.739 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:00 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:00.739 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:00 smithi105 ceph-mon[23707]: from='osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:26:00.740 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:26:00 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[44676]: 2024-09-13T22:26:00.635+0000 7f0949cd6700 -1 osd.3 0 waiting for initial osdmap 2024-09-13T22:26:00.740 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:26:00 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[44676]: 2024-09-13T22:26:00.643+0000 7f0946671700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:26:00.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:00 smithi118 ceph-mon[25717]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:26:00.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:00 smithi118 ceph-mon[25717]: from='osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T22:26:00.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:00 smithi118 ceph-mon[25717]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T22:26:00.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:00 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:00.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:00 smithi118 ceph-mon[25717]: from='osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:26:01.919 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:01 smithi105 ceph-mon[23707]: from='osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-13T22:26:01.919 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:01 smithi105 ceph-mon[23707]: osdmap e26: 4 total, 3 up, 4 in 2024-09-13T22:26:01.919 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:01 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:01.919 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:01 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:01.919 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:01 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:01.948 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:01 smithi118 ceph-mon[25717]: from='osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-13T22:26:01.948 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:01 smithi118 ceph-mon[25717]: osdmap e26: 4 total, 3 up, 4 in 2024-09-13T22:26:01.948 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:01 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:01.948 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:01 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:01.948 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:01 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:02.194 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:26:02.214 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi118:vg_nvme/lv_4 2024-09-13T22:26:02.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:02 smithi118 ceph-mon[25717]: purged_snaps scrub starts 2024-09-13T22:26:02.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:02 smithi118 ceph-mon[25717]: purged_snaps scrub ok 2024-09-13T22:26:02.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:02 smithi118 ceph-mon[25717]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:26:02.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:02 smithi118 ceph-mon[25717]: osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773] boot 2024-09-13T22:26:02.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:02 smithi118 ceph-mon[25717]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T22:26:02.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:02 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:02.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:02 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:02.880 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:02 smithi105 ceph-mon[23707]: purged_snaps scrub starts 2024-09-13T22:26:02.880 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:02 smithi105 ceph-mon[23707]: purged_snaps scrub ok 2024-09-13T22:26:02.880 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:02 smithi105 ceph-mon[23707]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T22:26:02.880 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:02 smithi105 ceph-mon[23707]: osd.3 [v2:172.21.15.105:6826/2537607773,v1:172.21.15.105:6827/2537607773] boot 2024-09-13T22:26:02.881 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:02 smithi105 ceph-mon[23707]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T22:26:02.881 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:02 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:26:02.881 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:02 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:03.979 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:03 smithi105 ceph-mon[23707]: osdmap e28: 4 total, 4 up, 4 in 2024-09-13T22:26:04.067 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:03 smithi118 ceph-mon[25717]: osdmap e28: 4 total, 4 up, 4 in 2024-09-13T22:26:05.073 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:04 smithi118 ceph-mon[25717]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:05.073 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:26:05.073 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:26:05.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:05.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:04 smithi105 ceph-mon[23707]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:05.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:26:05.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:26:05.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:05.752 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:26:05.752 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:05.752 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:05.753 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:05.753 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:05.753 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:05.753 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:05.753 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:05.753 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:05.753 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:05.753 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:05.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:26:05.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:05.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:05.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:05.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:05.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:05.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:05.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:05.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:05.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:05.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:06.985 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:06 smithi105 ceph-mon[23707]: Detected new or changed devices on smithi105 2024-09-13T22:26:06.986 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:06 smithi105 ceph-mon[23707]: Adjusting osd_memory_target on smithi105 to 3216M 2024-09-13T22:26:06.986 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:06 smithi105 ceph-mon[23707]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:06.986 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:06 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/4268257706' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0615dcdf-718a-42a7-b44f-a6007769f736"}]: dispatch 2024-09-13T22:26:06.986 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:06 smithi105 ceph-mon[23707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0615dcdf-718a-42a7-b44f-a6007769f736"}]: dispatch 2024-09-13T22:26:06.986 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:06 smithi105 ceph-mon[23707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0615dcdf-718a-42a7-b44f-a6007769f736"}]': finished 2024-09-13T22:26:06.986 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:06 smithi105 ceph-mon[23707]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T22:26:06.986 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:06 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:06 smithi118 ceph-mon[25717]: Detected new or changed devices on smithi105 2024-09-13T22:26:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:06 smithi118 ceph-mon[25717]: Adjusting osd_memory_target on smithi105 to 3216M 2024-09-13T22:26:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:06 smithi118 ceph-mon[25717]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:06 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/4268257706' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0615dcdf-718a-42a7-b44f-a6007769f736"}]: dispatch 2024-09-13T22:26:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:06 smithi118 ceph-mon[25717]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0615dcdf-718a-42a7-b44f-a6007769f736"}]: dispatch 2024-09-13T22:26:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:06 smithi118 ceph-mon[25717]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0615dcdf-718a-42a7-b44f-a6007769f736"}]': finished 2024-09-13T22:26:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:06 smithi118 ceph-mon[25717]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T22:26:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:06 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:07.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:07 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/541025208' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:26:08.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:07 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/541025208' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:26:09.008 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:08 smithi105 ceph-mon[23707]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:09.049 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:08 smithi118 ceph-mon[25717]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:10 smithi105 ceph-mon[23707]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:10 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:10 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:10.906 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:10 smithi118 ceph-mon[25717]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:10.906 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:10 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:10.906 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:10 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:12 smithi118 ceph-mon[25717]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T22:26:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:12.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:12 smithi105 ceph-mon[23707]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:12.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T22:26:12.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:13.766 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:13 smithi118 ceph-mon[25717]: Deploying daemon osd.4 on smithi118 2024-09-13T22:26:13.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:13 smithi105 ceph-mon[23707]: Deploying daemon osd.4 on smithi118 2024-09-13T22:26:14.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:14 smithi105 ceph-mon[23707]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:14.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:14 smithi118 ceph-mon[25717]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:16.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:16 smithi118 ceph-mon[25717]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:16.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:16 smithi105 ceph-mon[23707]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:18.436 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:18.436 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:18.437 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:18.437 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:18.437 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:18 smithi118 ceph-mon[25717]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:18.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:18.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:18.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:18.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:18.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:18 smithi105 ceph-mon[23707]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:20.662 INFO:teuthology.orchestra.run.smithi118.stdout:Created osd(s) 4 on host 'smithi118' 2024-09-13T22:26:20.904 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:20 smithi118 ceph-mon[25717]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:20.904 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:20 smithi118 ceph-mon[25717]: from='osd.4 [v2:172.21.15.118:6800/445314513,v1:172.21.15.118:6801/445314513]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T22:26:20.904 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:20 smithi118 ceph-mon[25717]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T22:26:21.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:20 smithi105 ceph-mon[23707]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:21.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:20 smithi105 ceph-mon[23707]: from='osd.4 [v2:172.21.15.118:6800/445314513,v1:172.21.15.118:6801/445314513]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T22:26:21.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:20 smithi105 ceph-mon[23707]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T22:26:21.513 DEBUG:teuthology.orchestra.run.smithi118:osd.4> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T22:26:21.515 INFO:tasks.cephadm:Deploying osd.5 on smithi118 with /dev/vg_nvme/lv_3... 2024-09-13T22:26:21.515 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T22:26:21.764 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:21 smithi118 ceph-mon[25717]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T22:26:21.764 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:21 smithi118 ceph-mon[25717]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T22:26:21.764 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:21 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:21.765 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:21 smithi118 ceph-mon[25717]: from='osd.4 [v2:172.21.15.118:6800/445314513,v1:172.21.15.118:6801/445314513]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:26:21.765 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:21 smithi118 ceph-mon[25717]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:26:21.765 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:21 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:21.765 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:21 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:21.765 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:26:21 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[29898]: 2024-09-13T22:26:21.661+0000 7fd0c842c700 -1 osd.4 0 waiting for initial osdmap 2024-09-13T22:26:21.765 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:26:21 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[29898]: 2024-09-13T22:26:21.666+0000 7fd0c4dc7700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:26:22.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:21 smithi105 ceph-mon[23707]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T22:26:22.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:21 smithi105 ceph-mon[23707]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T22:26:22.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:21 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:22.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:21 smithi105 ceph-mon[23707]: from='osd.4 [v2:172.21.15.118:6800/445314513,v1:172.21.15.118:6801/445314513]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:26:22.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:21 smithi105 ceph-mon[23707]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:26:22.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:21 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:22.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:21 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:22.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:22 smithi118 ceph-mon[25717]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:22.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:22 smithi118 ceph-mon[25717]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-13T22:26:22.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:22 smithi118 ceph-mon[25717]: osdmap e31: 5 total, 4 up, 5 in 2024-09-13T22:26:22.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:22 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:22.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:22 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:23.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:22 smithi105 ceph-mon[23707]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:23.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:22 smithi105 ceph-mon[23707]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-13T22:26:23.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:22 smithi105 ceph-mon[23707]: osdmap e31: 5 total, 4 up, 5 in 2024-09-13T22:26:23.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:22 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:23.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:22 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:24.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:23 smithi118 ceph-mon[25717]: purged_snaps scrub starts 2024-09-13T22:26:24.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:23 smithi118 ceph-mon[25717]: purged_snaps scrub ok 2024-09-13T22:26:24.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:23 smithi118 ceph-mon[25717]: osd.4 [v2:172.21.15.118:6800/445314513,v1:172.21.15.118:6801/445314513] boot 2024-09-13T22:26:24.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:23 smithi118 ceph-mon[25717]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T22:26:24.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:23 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:24.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:23 smithi118 ceph-mon[25717]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T22:26:24.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:23 smithi105 ceph-mon[23707]: purged_snaps scrub starts 2024-09-13T22:26:24.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:23 smithi105 ceph-mon[23707]: purged_snaps scrub ok 2024-09-13T22:26:24.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:23 smithi105 ceph-mon[23707]: osd.4 [v2:172.21.15.118:6800/445314513,v1:172.21.15.118:6801/445314513] boot 2024-09-13T22:26:24.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:23 smithi105 ceph-mon[23707]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T22:26:24.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:23 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:26:24.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:23 smithi105 ceph-mon[23707]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T22:26:24.898 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:24.898 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:24.898 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:24.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:24.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:24.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:24.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:24.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:24.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:24.899 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:24 smithi118 ceph-mon[25717]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T22:26:25.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T22:26:25.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:25.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:25.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:25.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:25.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:25.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:25.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:25.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:25.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:24 smithi105 ceph-mon[23707]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T22:26:25.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:25 smithi105 ceph-mon[23707]: Detected new or changed devices on smithi118 2024-09-13T22:26:25.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:25 smithi105 ceph-mon[23707]: Adjusting osd_memory_target on smithi118 to 15936M 2024-09-13T22:26:25.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:25 smithi105 ceph-mon[23707]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T22:26:25.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:26:25.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:26:25.945 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:25 smithi118 ceph-mon[25717]: Detected new or changed devices on smithi118 2024-09-13T22:26:25.945 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:25 smithi118 ceph-mon[25717]: Adjusting osd_memory_target on smithi118 to 15936M 2024-09-13T22:26:25.945 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:25 smithi118 ceph-mon[25717]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T22:26:25.945 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:26:25.945 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:26:25.949 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:26:25.969 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi118:vg_nvme/lv_3 2024-09-13T22:26:26.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:26 smithi118 ceph-mon[25717]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 131 KiB/s, 0 objects/s recovering 2024-09-13T22:26:26.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:26 smithi118 ceph-mon[25717]: osdmap e36: 5 total, 5 up, 5 in 2024-09-13T22:26:27.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:26 smithi105 ceph-mon[23707]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 131 KiB/s, 0 objects/s recovering 2024-09-13T22:26:27.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:26 smithi105 ceph-mon[23707]: osdmap e36: 5 total, 5 up, 5 in 2024-09-13T22:26:28.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:28 smithi118 ceph-mon[25717]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 121 KiB/s, 0 objects/s recovering 2024-09-13T22:26:28.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:28 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:26:28.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:28 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:26:28.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:28 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:28.942 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:28 smithi105 ceph-mon[23707]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 121 KiB/s, 0 objects/s recovering 2024-09-13T22:26:28.942 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:28 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:26:28.943 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:28 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:26:28.943 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:28 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:29.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:29 smithi118 ceph-mon[25717]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:26:30.135 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:29 smithi105 ceph-mon[23707]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:26:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:30 smithi118 ceph-mon[25717]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T22:26:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:30 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/3289111422' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11fdabc1-dbe9-44e7-ac3f-e6033181bb56"}]: dispatch 2024-09-13T22:26:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:30 smithi118 ceph-mon[25717]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11fdabc1-dbe9-44e7-ac3f-e6033181bb56"}]: dispatch 2024-09-13T22:26:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:30 smithi118 ceph-mon[25717]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "11fdabc1-dbe9-44e7-ac3f-e6033181bb56"}]': finished 2024-09-13T22:26:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:30 smithi118 ceph-mon[25717]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T22:26:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:31.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:30 smithi105 ceph-mon[23707]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T22:26:31.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:30 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/3289111422' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11fdabc1-dbe9-44e7-ac3f-e6033181bb56"}]: dispatch 2024-09-13T22:26:31.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:30 smithi105 ceph-mon[23707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11fdabc1-dbe9-44e7-ac3f-e6033181bb56"}]: dispatch 2024-09-13T22:26:31.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:30 smithi105 ceph-mon[23707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "11fdabc1-dbe9-44e7-ac3f-e6033181bb56"}]': finished 2024-09-13T22:26:31.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:30 smithi105 ceph-mon[23707]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T22:26:31.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:31 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/3028037937' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:26:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:31 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/3028037937' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:26:33.013 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:32 smithi118 ceph-mon[25717]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:33.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:32 smithi105 ceph-mon[23707]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:34.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:34 smithi118 ceph-mon[25717]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:34.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:34 smithi105 ceph-mon[23707]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:36.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:36 smithi118 ceph-mon[25717]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:36.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:36 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T22:26:36.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:36 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:37.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:36 smithi105 ceph-mon[23707]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:37.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:36 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T22:26:37.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:36 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:37.734 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:37 smithi118 ceph-mon[25717]: Deploying daemon osd.5 on smithi118 2024-09-13T22:26:38.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:37 smithi105 ceph-mon[23707]: Deploying daemon osd.5 on smithi118 2024-09-13T22:26:38.940 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:38 smithi118 ceph-mon[25717]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:39.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:38 smithi105 ceph-mon[23707]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:40.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:40 smithi118 ceph-mon[25717]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:41.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:40 smithi105 ceph-mon[23707]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:42.692 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:42.692 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:42.692 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:42.692 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:42.692 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:42 smithi105 ceph-mon[23707]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:42.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:42.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:42.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:42.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:42.743 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:42 smithi118 ceph-mon[25717]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:44.268 INFO:teuthology.orchestra.run.smithi118.stdout:Created osd(s) 5 on host 'smithi118' 2024-09-13T22:26:44.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:44 smithi105 ceph-mon[23707]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:44.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:44 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:44 smithi118 ceph-mon[25717]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:44 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:45.132 DEBUG:teuthology.orchestra.run.smithi118:osd.5> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T22:26:45.134 INFO:tasks.cephadm:Deploying osd.6 on smithi118 with /dev/vg_nvme/lv_2... 2024-09-13T22:26:45.134 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T22:26:45.495 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:45 smithi118 ceph-mon[25717]: from='osd.5 [v2:172.21.15.118:6808/3938918551,v1:172.21.15.118:6809/3938918551]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T22:26:45.495 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:45 smithi118 ceph-mon[25717]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T22:26:45.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:45 smithi105 ceph-mon[23707]: from='osd.5 [v2:172.21.15.118:6808/3938918551,v1:172.21.15.118:6809/3938918551]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T22:26:45.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:45 smithi105 ceph-mon[23707]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T22:26:46.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:46 smithi118 ceph-mon[25717]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T22:26:46.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:46 smithi118 ceph-mon[25717]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T22:26:46.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:46 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:46.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:46 smithi118 ceph-mon[25717]: from='osd.5 [v2:172.21.15.118:6808/3938918551,v1:172.21.15.118:6809/3938918551]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:26:46.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:46 smithi118 ceph-mon[25717]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:26:46.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:46 smithi118 ceph-mon[25717]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:46.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:46 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:46.598 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:26:46 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[32865]: 2024-09-13T22:26:46.374+0000 7f6f53221700 -1 osd.5 0 waiting for initial osdmap 2024-09-13T22:26:46.598 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:26:46 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[32865]: 2024-09-13T22:26:46.380+0000 7f6f4f3bb700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:26:46.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:46 smithi105 ceph-mon[23707]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T22:26:46.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:46 smithi105 ceph-mon[23707]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T22:26:46.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:46 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:46.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:46 smithi105 ceph-mon[23707]: from='osd.5 [v2:172.21.15.118:6808/3938918551,v1:172.21.15.118:6809/3938918551]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:26:46.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:46 smithi105 ceph-mon[23707]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:26:46.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:46 smithi105 ceph-mon[23707]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T22:26:46.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:46 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:47.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:47 smithi105 ceph-mon[23707]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-13T22:26:47.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:47 smithi105 ceph-mon[23707]: osdmap e39: 6 total, 5 up, 6 in 2024-09-13T22:26:47.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:47 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:47.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:47 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:47.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:47 smithi105 ceph-mon[23707]: osd.5 [v2:172.21.15.118:6808/3938918551,v1:172.21.15.118:6809/3938918551] boot 2024-09-13T22:26:47.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:47 smithi105 ceph-mon[23707]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T22:26:47.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:47 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:47 smithi118 ceph-mon[25717]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-13T22:26:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:47 smithi118 ceph-mon[25717]: osdmap e39: 6 total, 5 up, 6 in 2024-09-13T22:26:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:47 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:47 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:47 smithi118 ceph-mon[25717]: osd.5 [v2:172.21.15.118:6808/3938918551,v1:172.21.15.118:6809/3938918551] boot 2024-09-13T22:26:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:47 smithi118 ceph-mon[25717]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T22:26:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:47 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:26:48.488 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:48 smithi118 ceph-mon[25717]: purged_snaps scrub starts 2024-09-13T22:26:48.488 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:48 smithi118 ceph-mon[25717]: purged_snaps scrub ok 2024-09-13T22:26:48.488 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:48 smithi118 ceph-mon[25717]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:26:48.488 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:48 smithi118 ceph-mon[25717]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T22:26:48.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:48 smithi105 ceph-mon[23707]: purged_snaps scrub starts 2024-09-13T22:26:48.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:48 smithi105 ceph-mon[23707]: purged_snaps scrub ok 2024-09-13T22:26:48.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:48 smithi105 ceph-mon[23707]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:26:48.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:48 smithi105 ceph-mon[23707]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T22:26:49.379 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:26:49.400 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi118:vg_nvme/lv_2 2024-09-13T22:26:50.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:50 smithi105 ceph-mon[23707]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T22:26:50.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:50 smithi105 ceph-mon[23707]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:26:50.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:50 smithi118 ceph-mon[25717]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T22:26:50.732 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:50 smithi118 ceph-mon[25717]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:26:51.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:51 smithi118 ceph-mon[25717]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T22:26:51.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:51 smithi118 ceph-mon[25717]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T22:26:51.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:51 smithi105 ceph-mon[23707]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T22:26:51.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:51 smithi105 ceph-mon[23707]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T22:26:52.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:52 smithi105 ceph-mon[23707]: osdmap e44: 6 total, 6 up, 6 in 2024-09-13T22:26:52.646 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:52 smithi105 ceph-mon[23707]: from='client.24185 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:26:52.646 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:52 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:26:52.646 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:52 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:26:52.647 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:52 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:52.647 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:52 smithi105 ceph-mon[23707]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:26:52.675 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:52 smithi118 ceph-mon[25717]: osdmap e44: 6 total, 6 up, 6 in 2024-09-13T22:26:52.675 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:52 smithi118 ceph-mon[25717]: from='client.24185 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:26:52.675 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:52 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:26:52.676 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:52 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:26:52.676 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:52 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:52.676 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:52 smithi118 ceph-mon[25717]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:26:54.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: Detected new or changed devices on smithi118 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: Adjusting osd_memory_target on smithi118 to 7968M 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/2650802648' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fcbf9ca-b241-4a4c-8f2d-9f3bbedf1d2b"}]: dispatch 2024-09-13T22:26:54.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fcbf9ca-b241-4a4c-8f2d-9f3bbedf1d2b"}]: dispatch 2024-09-13T22:26:54.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fcbf9ca-b241-4a4c-8f2d-9f3bbedf1d2b"}]': finished 2024-09-13T22:26:54.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T22:26:54.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:54 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:26:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: Detected new or changed devices on smithi118 2024-09-13T22:26:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: Adjusting osd_memory_target on smithi118 to 7968M 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/2650802648' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fcbf9ca-b241-4a4c-8f2d-9f3bbedf1d2b"}]: dispatch 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fcbf9ca-b241-4a4c-8f2d-9f3bbedf1d2b"}]: dispatch 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fcbf9ca-b241-4a4c-8f2d-9f3bbedf1d2b"}]': finished 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T22:26:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:54 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:26:55.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:55 smithi105 ceph-mon[23707]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T22:26:55.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:55 smithi105 ceph-mon[23707]: Cluster is now healthy 2024-09-13T22:26:55.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:55 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/2152349546' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:26:55.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:55 smithi118 ceph-mon[25717]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T22:26:55.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:55 smithi118 ceph-mon[25717]: Cluster is now healthy 2024-09-13T22:26:55.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:55 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/2152349546' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:26:56.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:56 smithi105 ceph-mon[23707]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T22:26:56.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:56 smithi118 ceph-mon[25717]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T22:26:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:26:58 smithi118 ceph-mon[25717]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T22:26:59.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:26:58 smithi105 ceph-mon[23707]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T22:27:00.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:00 smithi118 ceph-mon[25717]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-09-13T22:27:01.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:00 smithi105 ceph-mon[23707]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-09-13T22:27:01.788 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:01 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T22:27:01.788 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:01 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:01.789 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:01 smithi118 ceph-mon[25717]: Deploying daemon osd.6 on smithi118 2024-09-13T22:27:02.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:01 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T22:27:02.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:01 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:02.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:01 smithi105 ceph-mon[23707]: Deploying daemon osd.6 on smithi118 2024-09-13T22:27:02.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:02 smithi118 ceph-mon[25717]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T22:27:03.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:02 smithi105 ceph-mon[23707]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T22:27:04.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:04 smithi105 ceph-mon[23707]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:04.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:04 smithi118 ceph-mon[25717]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:07.070 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:06 smithi118 ceph-mon[25717]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:07.071 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:06 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:07.071 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:06 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:27:07.071 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:06 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:07.071 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:06 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:27:07.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:06 smithi105 ceph-mon[23707]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:07.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:06 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:07.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:06 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:27:07.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:06 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:07.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:06 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:27:08.797 INFO:teuthology.orchestra.run.smithi118.stdout:Created osd(s) 6 on host 'smithi118' 2024-09-13T22:27:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:08 smithi118 ceph-mon[25717]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:09.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:08 smithi105 ceph-mon[23707]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:09.458 DEBUG:teuthology.orchestra.run.smithi118:osd.6> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T22:27:09.461 INFO:tasks.cephadm:Deploying osd.7 on smithi118 with /dev/vg_nvme/lv_1... 2024-09-13T22:27:09.461 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T22:27:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:09 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:09 smithi118 ceph-mon[25717]: from='osd.6 [v2:172.21.15.118:6816/38252558,v1:172.21.15.118:6817/38252558]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T22:27:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:09 smithi118 ceph-mon[25717]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T22:27:10.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:09 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:10.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:09 smithi105 ceph-mon[23707]: from='osd.6 [v2:172.21.15.118:6816/38252558,v1:172.21.15.118:6817/38252558]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T22:27:10.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:09 smithi105 ceph-mon[23707]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T22:27:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:10 smithi118 ceph-mon[25717]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:10 smithi118 ceph-mon[25717]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T22:27:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:10 smithi118 ceph-mon[25717]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T22:27:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:10 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:27:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:10 smithi118 ceph-mon[25717]: from='osd.6 [v2:172.21.15.118:6816/38252558,v1:172.21.15.118:6817/38252558]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:27:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:10 smithi118 ceph-mon[25717]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:27:10.848 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:27:10 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[35838]: 2024-09-13T22:27:10.806+0000 7f8c69225700 -1 osd.6 0 waiting for initial osdmap 2024-09-13T22:27:10.848 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:27:10 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[35838]: 2024-09-13T22:27:10.812+0000 7f8c633bb700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:27:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:10 smithi105 ceph-mon[23707]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:10 smithi105 ceph-mon[23707]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T22:27:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:10 smithi105 ceph-mon[23707]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T22:27:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:10 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:27:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:10 smithi105 ceph-mon[23707]: from='osd.6 [v2:172.21.15.118:6816/38252558,v1:172.21.15.118:6817/38252558]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:27:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:10 smithi105 ceph-mon[23707]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:27:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:11 smithi118 ceph-mon[25717]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-13T22:27:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:11 smithi118 ceph-mon[25717]: osdmap e47: 7 total, 6 up, 7 in 2024-09-13T22:27:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:11 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:27:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:11 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:27:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:11 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:12.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:11 smithi105 ceph-mon[23707]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-13T22:27:12.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:11 smithi105 ceph-mon[23707]: osdmap e47: 7 total, 6 up, 7 in 2024-09-13T22:27:12.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:11 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:27:12.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:11 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:27:12.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:11 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:12 smithi118 ceph-mon[25717]: purged_snaps scrub starts 2024-09-13T22:27:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:12 smithi118 ceph-mon[25717]: purged_snaps scrub ok 2024-09-13T22:27:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:12 smithi118 ceph-mon[25717]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:12 smithi118 ceph-mon[25717]: osd.6 [v2:172.21.15.118:6816/38252558,v1:172.21.15.118:6817/38252558] boot 2024-09-13T22:27:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:12 smithi118 ceph-mon[25717]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T22:27:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:27:13.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:12 smithi105 ceph-mon[23707]: purged_snaps scrub starts 2024-09-13T22:27:13.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:12 smithi105 ceph-mon[23707]: purged_snaps scrub ok 2024-09-13T22:27:13.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:12 smithi105 ceph-mon[23707]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T22:27:13.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:12 smithi105 ceph-mon[23707]: osd.6 [v2:172.21.15.118:6816/38252558,v1:172.21.15.118:6817/38252558] boot 2024-09-13T22:27:13.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:12 smithi105 ceph-mon[23707]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T22:27:13.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:27:13.469 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-13T22:27:13.493 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi118:vg_nvme/lv_1 2024-09-13T22:27:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:13 smithi118 ceph-mon[25717]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T22:27:14.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:13 smithi105 ceph-mon[23707]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T22:27:15.024 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:14 smithi118 ceph-mon[25717]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:15.024 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:14 smithi118 ceph-mon[25717]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T22:27:15.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:14 smithi105 ceph-mon[23707]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:15.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:14 smithi105 ceph-mon[23707]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:16.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:27:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T22:27:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:27:16.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:16.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:27:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:16 smithi118 ceph-mon[25717]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:16 smithi118 ceph-mon[25717]: Detected new or changed devices on smithi118 2024-09-13T22:27:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:16 smithi118 ceph-mon[25717]: Adjusting osd_memory_target on smithi118 to 5312M 2024-09-13T22:27:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:16 smithi118 ceph-mon[25717]: osdmap e52: 7 total, 7 up, 7 in 2024-09-13T22:27:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:17.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:27:17.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:27:17.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:17.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:16 smithi105 ceph-mon[23707]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:17.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:16 smithi105 ceph-mon[23707]: Detected new or changed devices on smithi118 2024-09-13T22:27:17.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:16 smithi105 ceph-mon[23707]: Adjusting osd_memory_target on smithi118 to 5312M 2024-09-13T22:27:17.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:16 smithi105 ceph-mon[23707]: osdmap e52: 7 total, 7 up, 7 in 2024-09-13T22:27:17.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:17.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T22:27:17.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T22:27:17.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:18.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:17 smithi118 ceph-mon[25717]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:27:18.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:17 smithi105 ceph-mon[23707]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:27:19.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:18 smithi118 ceph-mon[25717]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:19.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:18 smithi105 ceph-mon[23707]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:20.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:19 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/2709722440' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "319cc1f5-41b7-4cbb-9b84-8263613da97b"}]: dispatch 2024-09-13T22:27:20.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:19 smithi118 ceph-mon[25717]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "319cc1f5-41b7-4cbb-9b84-8263613da97b"}]: dispatch 2024-09-13T22:27:20.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:19 smithi118 ceph-mon[25717]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "319cc1f5-41b7-4cbb-9b84-8263613da97b"}]': finished 2024-09-13T22:27:20.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:19 smithi118 ceph-mon[25717]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T22:27:20.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:19 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:20.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:19 smithi118 ceph-mon[25717]: from='client.? 172.21.15.118:0/582750385' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:27:20.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:19 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/2709722440' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "319cc1f5-41b7-4cbb-9b84-8263613da97b"}]: dispatch 2024-09-13T22:27:20.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:19 smithi105 ceph-mon[23707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "319cc1f5-41b7-4cbb-9b84-8263613da97b"}]: dispatch 2024-09-13T22:27:20.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:19 smithi105 ceph-mon[23707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "319cc1f5-41b7-4cbb-9b84-8263613da97b"}]': finished 2024-09-13T22:27:20.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:19 smithi105 ceph-mon[23707]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T22:27:20.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:19 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:20.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:19 smithi105 ceph-mon[23707]: from='client.? 172.21.15.118:0/582750385' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T22:27:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:20 smithi118 ceph-mon[25717]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-13T22:27:21.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:20 smithi105 ceph-mon[23707]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-13T22:27:23.095 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:22 smithi118 ceph-mon[25717]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-13T22:27:23.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:22 smithi105 ceph-mon[23707]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-13T22:27:24.446 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:24 smithi118 ceph-mon[25717]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T22:27:24.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:24 smithi105 ceph-mon[23707]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T22:27:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T22:27:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:25.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T22:27:25.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:26.823 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:26 smithi118 ceph-mon[25717]: Deploying daemon osd.7 on smithi118 2024-09-13T22:27:26.824 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:26 smithi118 ceph-mon[25717]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-13T22:27:26.824 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:26 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:27:26.824 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:26 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:27:26.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:26 smithi105 ceph-mon[23707]: Deploying daemon osd.7 on smithi118 2024-09-13T22:27:26.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:26 smithi105 ceph-mon[23707]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-13T22:27:26.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:27:26.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:27:28.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:28 smithi118 ceph-mon[25717]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T22:27:29.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:28 smithi105 ceph-mon[23707]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T22:27:30.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:30 smithi118 ceph-mon[25717]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:30.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:30.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:27:30.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:30.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:27:30.942 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:30 smithi105 ceph-mon[23707]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:30.942 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:30.942 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:27:30.942 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:30.942 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:27:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:32 smithi118 ceph-mon[25717]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:33.119 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:32 smithi105 ceph-mon[23707]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:33.613 INFO:teuthology.orchestra.run.smithi118.stdout:Created osd(s) 7 on host 'smithi118' 2024-09-13T22:27:34.289 DEBUG:teuthology.orchestra.run.smithi118:osd.7> sudo journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T22:27:34.291 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T22:27:34.292 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T22:27:34.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:34 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:34.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:34 smithi105 ceph-mon[23707]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:34.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:34 smithi105 ceph-mon[23707]: from='osd.7 [v2:172.21.15.118:6824/2953607150,v1:172.21.15.118:6825/2953607150]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T22:27:34.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:34 smithi105 ceph-mon[23707]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T22:27:34.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:34 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:34.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:34 smithi118 ceph-mon[25717]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:34.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:34 smithi118 ceph-mon[25717]: from='osd.7 [v2:172.21.15.118:6824/2953607150,v1:172.21.15.118:6825/2953607150]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T22:27:34.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:34 smithi118 ceph-mon[25717]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T22:27:35.749 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:35 smithi118 ceph-mon[25717]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T22:27:35.749 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:35 smithi118 ceph-mon[25717]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T22:27:35.749 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:35 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:35.749 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:35 smithi118 ceph-mon[25717]: from='osd.7 [v2:172.21.15.118:6824/2953607150,v1:172.21.15.118:6825/2953607150]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:27:35.749 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:35 smithi118 ceph-mon[25717]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:27:35.749 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:35 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:35.750 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:27:35 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[38904]: 2024-09-13T22:27:35.624+0000 7f5035a9a700 -1 osd.7 0 waiting for initial osdmap 2024-09-13T22:27:35.750 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:27:35 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[38904]: 2024-09-13T22:27:35.631+0000 7f502f42f700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:27:35.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:35 smithi105 ceph-mon[23707]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T22:27:35.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:35 smithi105 ceph-mon[23707]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T22:27:35.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:35 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:35.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:35 smithi105 ceph-mon[23707]: from='osd.7 [v2:172.21.15.118:6824/2953607150,v1:172.21.15.118:6825/2953607150]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:27:35.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:35 smithi105 ceph-mon[23707]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:27:35.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:35 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:36.505 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:27:36.754 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:36 smithi118 ceph-mon[25717]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:36.754 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:36 smithi118 ceph-mon[25717]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-13T22:27:36.754 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:36 smithi118 ceph-mon[25717]: osdmap e55: 8 total, 7 up, 8 in 2024-09-13T22:27:36.754 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:36 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:36.754 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:36 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:36.755 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:36 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3391758793' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T22:27:36.755 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:36 smithi118 ceph-mon[25717]: osd.7 [v2:172.21.15.118:6824/2953607150,v1:172.21.15.118:6825/2953607150] boot 2024-09-13T22:27:36.755 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:36 smithi118 ceph-mon[25717]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T22:27:36.755 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:36 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:36.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:36 smithi105 ceph-mon[23707]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T22:27:36.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:36 smithi105 ceph-mon[23707]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-13T22:27:36.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:36 smithi105 ceph-mon[23707]: osdmap e55: 8 total, 7 up, 8 in 2024-09-13T22:27:36.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:36 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:36.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:36 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:36.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:36 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3391758793' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T22:27:36.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:36 smithi105 ceph-mon[23707]: osd.7 [v2:172.21.15.118:6824/2953607150,v1:172.21.15.118:6825/2953607150] boot 2024-09-13T22:27:36.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:36 smithi105 ceph-mon[23707]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T22:27:36.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:36 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:27:37.212 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":55,"num_osds":8,"num_up_osds":7,"osd_up_since":1726266431,"num_in_osds":8,"osd_in_since":1726266439,"num_remapped_pgs":0} 2024-09-13T22:27:37.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:37 smithi118 ceph-mon[25717]: purged_snaps scrub starts 2024-09-13T22:27:37.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:37 smithi118 ceph-mon[25717]: purged_snaps scrub ok 2024-09-13T22:27:37.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:37 smithi118 ceph-mon[25717]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T22:27:37.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:37 smithi105 ceph-mon[23707]: purged_snaps scrub starts 2024-09-13T22:27:37.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:37 smithi105 ceph-mon[23707]: purged_snaps scrub ok 2024-09-13T22:27:37.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:37 smithi105 ceph-mon[23707]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T22:27:38.214 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T22:27:38.857 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:38.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:38 smithi105 ceph-mon[23707]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T22:27:39.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:27:39.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:39.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:39.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:39.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:39.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:27:39.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:39.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:39.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:27:39.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:27:39.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:27:39.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:27:39.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:38 smithi118 ceph-mon[25717]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T22:27:39.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:39 smithi105 ceph-mon[23707]: Detected new or changed devices on smithi118 2024-09-13T22:27:39.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:39 smithi105 ceph-mon[23707]: Adjusting osd_memory_target on smithi118 to 3984M 2024-09-13T22:27:39.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:39 smithi105 ceph-mon[23707]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T22:27:40.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:39 smithi118 ceph-mon[25717]: Detected new or changed devices on smithi118 2024-09-13T22:27:40.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:39 smithi118 ceph-mon[25717]: Adjusting osd_memory_target on smithi118 to 3984M 2024-09-13T22:27:40.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:39 smithi118 ceph-mon[25717]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T22:27:40.257 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:27:40.834 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:40 smithi105 ceph-mon[23707]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:27:40.834 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:40 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/309432086' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T22:27:40.834 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:40 smithi105 ceph-mon[23707]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T22:27:40.857 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1726266456,"num_in_osds":8,"osd_in_since":1726266439,"num_remapped_pgs":0} 2024-09-13T22:27:40.858 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T22:27:41.062 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:40 smithi118 ceph-mon[25717]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:27:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:40 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/309432086' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T22:27:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:40 smithi118 ceph-mon[25717]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T22:27:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:42 smithi105 ceph-mon[23707]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:27:42.943 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:27:42.943 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":60,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","created":"2024-09-13T22:21:33.376975+0000","modified":"2024-09-13T22:27:40.618670+0000","last_up_change":"2024-09-13T22:27:36.613119+0000","last_in_change":"2024-09-13T22:27:19.189044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T22:25:35.865809+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b8b91105-6278-4f62-8a38-6a9b2964f21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6803","nonce":770182744}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6805","nonce":770182744}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6809","nonce":770182744}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6807","nonce":770182744}]},"public_addr":"172.21.15.105:6803/770182744","cluster_addr":"172.21.15.105:6805/770182744","heartbeat_back_addr":"172.21.15.105:6809/770182744","heartbeat_front_addr":"172.21.15.105:6807/770182744","state":["exists","up"]},{"osd":1,"uuid":"49d94541-03ae-40a1-9fcd-abd7730bc069","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6811","nonce":996806047}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6813","nonce":996806047}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6817","nonce":996806047}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6815","nonce":996806047}]},"public_addr":"172.21.15.105:6811/996806047","cluster_addr":"172.21.15.105:6813/996806047","heartbeat_back_addr":"172.21.15.105:6817/996806047","heartbeat_front_addr":"172.21.15.105:6815/996806047","state":["exists","up"]},{"osd":2,"uuid":"123dd0dc-4fd0-4620-8f36-6e63822846ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6819","nonce":1795261821}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6821","nonce":1795261821}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6825","nonce":1795261821}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6823","nonce":1795261821}]},"public_addr":"172.21.15.105:6819/1795261821","cluster_addr":"172.21.15.105:6821/1795261821","heartbeat_back_addr":"172.21.15.105:6825/1795261821","heartbeat_front_addr":"172.21.15.105:6823/1795261821","state":["exists","up"]},{"osd":3,"uuid":"bacf1e21-97ba-4157-8d3f-537c851503cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6827","nonce":2537607773}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6829","nonce":2537607773}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6833","nonce":2537607773}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6831","nonce":2537607773}]},"public_addr":"172.21.15.105:6827/2537607773","cluster_addr":"172.21.15.105:6829/2537607773","heartbeat_back_addr":"172.21.15.105:6833/2537607773","heartbeat_front_addr":"172.21.15.105:6831/2537607773","state":["exists","up"]},{"osd":4,"uuid":"0615dcdf-718a-42a7-b44f-a6007769f736","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6800","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6801","nonce":445314513}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6802","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6803","nonce":445314513}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6806","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6807","nonce":445314513}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6804","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6805","nonce":445314513}]},"public_addr":"172.21.15.118:6801/445314513","cluster_addr":"172.21.15.118:6803/445314513","heartbeat_back_addr":"172.21.15.118:6807/445314513","heartbeat_front_addr":"172.21.15.118:6805/445314513","state":["exists","up"]},{"osd":5,"uuid":"11fdabc1-dbe9-44e7-ac3f-e6033181bb56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6808","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6809","nonce":3938918551}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6810","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6811","nonce":3938918551}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6814","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6815","nonce":3938918551}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6812","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6813","nonce":3938918551}]},"public_addr":"172.21.15.118:6809/3938918551","cluster_addr":"172.21.15.118:6811/3938918551","heartbeat_back_addr":"172.21.15.118:6815/3938918551","heartbeat_front_addr":"172.21.15.118:6813/3938918551","state":["exists","up"]},{"osd":6,"uuid":"9fcbf9ca-b241-4a4c-8f2d-9f3bbedf1d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6816","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6817","nonce":38252558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6818","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6819","nonce":38252558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6822","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6823","nonce":38252558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6820","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6821","nonce":38252558}]},"public_addr":"172.21.15.118:6817/38252558","cluster_addr":"172.21.15.118:6819/38252558","heartbeat_back_addr":"172.21.15.118:6823/38252558","heartbeat_front_addr":"172.21.15.118:6821/38252558","state":["exists","up"]},{"osd":7,"uuid":"319cc1f5-41b7-4cbb-9b84-8263613da97b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6824","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6825","nonce":2953607150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6826","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6827","nonce":2953607150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6830","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6831","nonce":2953607150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6828","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6829","nonce":2953607150}]},"public_addr":"172.21.15.118:6825/2953607150","cluster_addr":"172.21.15.118:6827/2953607150","heartbeat_back_addr":"172.21.15.118:6831/2953607150","heartbeat_front_addr":"172.21.15.118:6829/2953607150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:24:35.253612+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:25:01.986625+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:25:32.393983+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:25:59.746771+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:26:21.230660+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:26:45.510288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:27:09.828435+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:27:34.634086+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:0/1995074497":"2024-09-14T22:23:24.785743+0000","172.21.15.105:6801/2551736598":"2024-09-14T22:23:24.785743+0000","172.21.15.105:6801/1612887222":"2024-09-14T22:22:34.979142+0000","172.21.15.105:6800/1612887222":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/2642202029":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/1787844468":"2024-09-14T22:23:24.785743+0000","172.21.15.105:0/2454599807":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/238887747":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/2617814584":"2024-09-14T22:23:24.785743+0000","172.21.15.105:6800/2551736598":"2024-09-14T22:23:24.785743+0000","172.21.15.105:0/3934146309":"2024-09-14T22:22:05.434535+0000","172.21.15.105:6800/1461283344":"2024-09-14T22:22:05.434535+0000","172.21.15.105:6801/1461283344":"2024-09-14T22:22:05.434535+0000","172.21.15.105:0/914342762":"2024-09-14T22:22:05.434535+0000","172.21.15.105:0/2000600795":"2024-09-14T22:22:05.434535+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T22:27:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:42 smithi118 ceph-mon[25717]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:27:43.715 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T22:25:35.865809+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T22:27:43.716 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T22:27:43.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:43 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1371667108' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T22:27:43.928 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:44.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/1371667108' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T22:27:44.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:44 smithi105 ceph-mon[23707]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T22:27:44.896 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:44 smithi118 ceph-mon[25717]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T22:27:45.767 INFO:teuthology.orchestra.run.smithi105.stdout:pg_num: 1 2024-09-13T22:27:46.518 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T22:27:46.518 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T22:27:46.817 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:46 smithi105 ceph-mon[23707]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T22:27:46.817 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:46 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/636576198' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T22:27:47.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:46 smithi118 ceph-mon[25717]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-13T22:27:47.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:46 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/636576198' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T22:27:48.691 INFO:teuthology.orchestra.run.smithi105.stdout:[client.0] 2024-09-13T22:27:48.692 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQBkvORmMSvjKBAAcsPA0J4Ssd/jP0kRv2lj9A== 2024-09-13T22:27:49.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:48 smithi118 ceph-mon[25717]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T22:27:49.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:48 smithi105 ceph-mon[23707]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T22:27:49.458 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-13T22:27:49.458 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T22:27:49.458 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T22:27:49.494 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T22:27:49.495 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T22:27:49.495 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T22:27:49.741 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:49.860 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:49 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3271969045' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T22:27:49.860 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:49 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3271969045' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T22:27:50.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:49 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3271969045' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T22:27:50.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:49 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3271969045' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T22:27:50.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:50 smithi105 ceph-mon[23707]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T22:27:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:50 smithi118 ceph-mon[25717]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T22:27:51.668 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:27:51.988 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:51 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/618253747' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T22:27:52.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:51 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/618253747' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T22:27:52.405 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi105.swmmea","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":3717692601},{"type":"v1","addr":"172.21.15.105:6801","nonce":3717692601}]},"active_addr":"172.21.15.105:6801/3717692601","active_change":"2024-09-13T22:23:24.786077+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi118.rbhtgz","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.105:8443/","prometheus":"http://172.21.15.105:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":1969433361}]},{"addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":1126077726}]},{"addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":3353763154}]},{"addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":261509613}]}]}} 2024-09-13T22:27:52.411 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T22:27:52.411 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T22:27:52.411 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T22:27:52.617 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:53.017 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:52 smithi105 ceph-mon[23707]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-09-13T22:27:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:52 smithi118 ceph-mon[25717]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-09-13T22:27:54.463 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:27:54.463 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":60,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","created":"2024-09-13T22:21:33.376975+0000","modified":"2024-09-13T22:27:40.618670+0000","last_up_change":"2024-09-13T22:27:36.613119+0000","last_in_change":"2024-09-13T22:27:19.189044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T22:25:35.865809+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b8b91105-6278-4f62-8a38-6a9b2964f21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6803","nonce":770182744}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6805","nonce":770182744}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6809","nonce":770182744}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6807","nonce":770182744}]},"public_addr":"172.21.15.105:6803/770182744","cluster_addr":"172.21.15.105:6805/770182744","heartbeat_back_addr":"172.21.15.105:6809/770182744","heartbeat_front_addr":"172.21.15.105:6807/770182744","state":["exists","up"]},{"osd":1,"uuid":"49d94541-03ae-40a1-9fcd-abd7730bc069","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6811","nonce":996806047}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6813","nonce":996806047}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6817","nonce":996806047}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6815","nonce":996806047}]},"public_addr":"172.21.15.105:6811/996806047","cluster_addr":"172.21.15.105:6813/996806047","heartbeat_back_addr":"172.21.15.105:6817/996806047","heartbeat_front_addr":"172.21.15.105:6815/996806047","state":["exists","up"]},{"osd":2,"uuid":"123dd0dc-4fd0-4620-8f36-6e63822846ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6819","nonce":1795261821}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6821","nonce":1795261821}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6825","nonce":1795261821}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6823","nonce":1795261821}]},"public_addr":"172.21.15.105:6819/1795261821","cluster_addr":"172.21.15.105:6821/1795261821","heartbeat_back_addr":"172.21.15.105:6825/1795261821","heartbeat_front_addr":"172.21.15.105:6823/1795261821","state":["exists","up"]},{"osd":3,"uuid":"bacf1e21-97ba-4157-8d3f-537c851503cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6827","nonce":2537607773}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6829","nonce":2537607773}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6833","nonce":2537607773}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6831","nonce":2537607773}]},"public_addr":"172.21.15.105:6827/2537607773","cluster_addr":"172.21.15.105:6829/2537607773","heartbeat_back_addr":"172.21.15.105:6833/2537607773","heartbeat_front_addr":"172.21.15.105:6831/2537607773","state":["exists","up"]},{"osd":4,"uuid":"0615dcdf-718a-42a7-b44f-a6007769f736","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6800","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6801","nonce":445314513}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6802","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6803","nonce":445314513}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6806","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6807","nonce":445314513}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6804","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6805","nonce":445314513}]},"public_addr":"172.21.15.118:6801/445314513","cluster_addr":"172.21.15.118:6803/445314513","heartbeat_back_addr":"172.21.15.118:6807/445314513","heartbeat_front_addr":"172.21.15.118:6805/445314513","state":["exists","up"]},{"osd":5,"uuid":"11fdabc1-dbe9-44e7-ac3f-e6033181bb56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6808","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6809","nonce":3938918551}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6810","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6811","nonce":3938918551}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6814","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6815","nonce":3938918551}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6812","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6813","nonce":3938918551}]},"public_addr":"172.21.15.118:6809/3938918551","cluster_addr":"172.21.15.118:6811/3938918551","heartbeat_back_addr":"172.21.15.118:6815/3938918551","heartbeat_front_addr":"172.21.15.118:6813/3938918551","state":["exists","up"]},{"osd":6,"uuid":"9fcbf9ca-b241-4a4c-8f2d-9f3bbedf1d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6816","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6817","nonce":38252558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6818","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6819","nonce":38252558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6822","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6823","nonce":38252558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6820","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6821","nonce":38252558}]},"public_addr":"172.21.15.118:6817/38252558","cluster_addr":"172.21.15.118:6819/38252558","heartbeat_back_addr":"172.21.15.118:6823/38252558","heartbeat_front_addr":"172.21.15.118:6821/38252558","state":["exists","up"]},{"osd":7,"uuid":"319cc1f5-41b7-4cbb-9b84-8263613da97b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6824","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6825","nonce":2953607150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6826","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6827","nonce":2953607150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6830","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6831","nonce":2953607150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6828","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6829","nonce":2953607150}]},"public_addr":"172.21.15.118:6825/2953607150","cluster_addr":"172.21.15.118:6827/2953607150","heartbeat_back_addr":"172.21.15.118:6831/2953607150","heartbeat_front_addr":"172.21.15.118:6829/2953607150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:24:35.253612+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:25:01.986625+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:25:32.393983+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:25:59.746771+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:26:21.230660+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:26:45.510288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:27:09.828435+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:27:34.634086+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:0/1995074497":"2024-09-14T22:23:24.785743+0000","172.21.15.105:6801/2551736598":"2024-09-14T22:23:24.785743+0000","172.21.15.105:6801/1612887222":"2024-09-14T22:22:34.979142+0000","172.21.15.105:6800/1612887222":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/2642202029":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/1787844468":"2024-09-14T22:23:24.785743+0000","172.21.15.105:0/2454599807":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/238887747":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/2617814584":"2024-09-14T22:23:24.785743+0000","172.21.15.105:6800/2551736598":"2024-09-14T22:23:24.785743+0000","172.21.15.105:0/3934146309":"2024-09-14T22:22:05.434535+0000","172.21.15.105:6800/1461283344":"2024-09-14T22:22:05.434535+0000","172.21.15.105:6801/1461283344":"2024-09-14T22:22:05.434535+0000","172.21.15.105:0/914342762":"2024-09-14T22:22:05.434535+0000","172.21.15.105:0/2000600795":"2024-09-14T22:22:05.434535+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T22:27:54.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:54 smithi105 ceph-mon[23707]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-13T22:27:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:54 smithi118 ceph-mon[25717]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-13T22:27:55.076 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T22:27:55.076 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T22:27:55.282 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:55.621 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:55 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/523958352' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T22:27:55.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:55 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/523958352' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T22:27:56.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:56 smithi105 ceph-mon[23707]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:27:56.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:56 smithi118 ceph-mon[25717]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:27:57.208 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:27:57.209 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":60,"fsid":"782bdfc8-721e-11ef-bceb-c7b262605968","created":"2024-09-13T22:21:33.376975+0000","modified":"2024-09-13T22:27:40.618670+0000","last_up_change":"2024-09-13T22:27:36.613119+0000","last_in_change":"2024-09-13T22:27:19.189044+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T22:25:35.865809+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b8b91105-6278-4f62-8a38-6a9b2964f21b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6803","nonce":770182744}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6805","nonce":770182744}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6809","nonce":770182744}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":770182744},{"type":"v1","addr":"172.21.15.105:6807","nonce":770182744}]},"public_addr":"172.21.15.105:6803/770182744","cluster_addr":"172.21.15.105:6805/770182744","heartbeat_back_addr":"172.21.15.105:6809/770182744","heartbeat_front_addr":"172.21.15.105:6807/770182744","state":["exists","up"]},{"osd":1,"uuid":"49d94541-03ae-40a1-9fcd-abd7730bc069","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6811","nonce":996806047}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6813","nonce":996806047}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6817","nonce":996806047}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":996806047},{"type":"v1","addr":"172.21.15.105:6815","nonce":996806047}]},"public_addr":"172.21.15.105:6811/996806047","cluster_addr":"172.21.15.105:6813/996806047","heartbeat_back_addr":"172.21.15.105:6817/996806047","heartbeat_front_addr":"172.21.15.105:6815/996806047","state":["exists","up"]},{"osd":2,"uuid":"123dd0dc-4fd0-4620-8f36-6e63822846ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6819","nonce":1795261821}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6821","nonce":1795261821}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6825","nonce":1795261821}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":1795261821},{"type":"v1","addr":"172.21.15.105:6823","nonce":1795261821}]},"public_addr":"172.21.15.105:6819/1795261821","cluster_addr":"172.21.15.105:6821/1795261821","heartbeat_back_addr":"172.21.15.105:6825/1795261821","heartbeat_front_addr":"172.21.15.105:6823/1795261821","state":["exists","up"]},{"osd":3,"uuid":"bacf1e21-97ba-4157-8d3f-537c851503cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6827","nonce":2537607773}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6829","nonce":2537607773}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6833","nonce":2537607773}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":2537607773},{"type":"v1","addr":"172.21.15.105:6831","nonce":2537607773}]},"public_addr":"172.21.15.105:6827/2537607773","cluster_addr":"172.21.15.105:6829/2537607773","heartbeat_back_addr":"172.21.15.105:6833/2537607773","heartbeat_front_addr":"172.21.15.105:6831/2537607773","state":["exists","up"]},{"osd":4,"uuid":"0615dcdf-718a-42a7-b44f-a6007769f736","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6800","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6801","nonce":445314513}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6802","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6803","nonce":445314513}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6806","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6807","nonce":445314513}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6804","nonce":445314513},{"type":"v1","addr":"172.21.15.118:6805","nonce":445314513}]},"public_addr":"172.21.15.118:6801/445314513","cluster_addr":"172.21.15.118:6803/445314513","heartbeat_back_addr":"172.21.15.118:6807/445314513","heartbeat_front_addr":"172.21.15.118:6805/445314513","state":["exists","up"]},{"osd":5,"uuid":"11fdabc1-dbe9-44e7-ac3f-e6033181bb56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6808","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6809","nonce":3938918551}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6810","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6811","nonce":3938918551}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6814","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6815","nonce":3938918551}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6812","nonce":3938918551},{"type":"v1","addr":"172.21.15.118:6813","nonce":3938918551}]},"public_addr":"172.21.15.118:6809/3938918551","cluster_addr":"172.21.15.118:6811/3938918551","heartbeat_back_addr":"172.21.15.118:6815/3938918551","heartbeat_front_addr":"172.21.15.118:6813/3938918551","state":["exists","up"]},{"osd":6,"uuid":"9fcbf9ca-b241-4a4c-8f2d-9f3bbedf1d2b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6816","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6817","nonce":38252558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6818","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6819","nonce":38252558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6822","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6823","nonce":38252558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6820","nonce":38252558},{"type":"v1","addr":"172.21.15.118:6821","nonce":38252558}]},"public_addr":"172.21.15.118:6817/38252558","cluster_addr":"172.21.15.118:6819/38252558","heartbeat_back_addr":"172.21.15.118:6823/38252558","heartbeat_front_addr":"172.21.15.118:6821/38252558","state":["exists","up"]},{"osd":7,"uuid":"319cc1f5-41b7-4cbb-9b84-8263613da97b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6824","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6825","nonce":2953607150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6826","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6827","nonce":2953607150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6830","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6831","nonce":2953607150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6828","nonce":2953607150},{"type":"v1","addr":"172.21.15.118:6829","nonce":2953607150}]},"public_addr":"172.21.15.118:6825/2953607150","cluster_addr":"172.21.15.118:6827/2953607150","heartbeat_back_addr":"172.21.15.118:6831/2953607150","heartbeat_front_addr":"172.21.15.118:6829/2953607150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:24:35.253612+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:25:01.986625+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:25:32.393983+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:25:59.746771+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:26:21.230660+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:26:45.510288+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:27:09.828435+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T22:27:34.634086+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:0/1995074497":"2024-09-14T22:23:24.785743+0000","172.21.15.105:6801/2551736598":"2024-09-14T22:23:24.785743+0000","172.21.15.105:6801/1612887222":"2024-09-14T22:22:34.979142+0000","172.21.15.105:6800/1612887222":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/2642202029":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/1787844468":"2024-09-14T22:23:24.785743+0000","172.21.15.105:0/2454599807":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/238887747":"2024-09-14T22:22:34.979142+0000","172.21.15.105:0/2617814584":"2024-09-14T22:23:24.785743+0000","172.21.15.105:6800/2551736598":"2024-09-14T22:23:24.785743+0000","172.21.15.105:0/3934146309":"2024-09-14T22:22:05.434535+0000","172.21.15.105:6800/1461283344":"2024-09-14T22:22:05.434535+0000","172.21.15.105:6801/1461283344":"2024-09-14T22:22:05.434535+0000","172.21.15.105:0/914342762":"2024-09-14T22:22:05.434535+0000","172.21.15.105:0/2000600795":"2024-09-14T22:22:05.434535+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T22:27:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:57 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/1718007451' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T22:27:57.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:57 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1718007451' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T22:27:57.965 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T22:27:57.966 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T22:27:57.966 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T22:27:57.966 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T22:27:57.967 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T22:27:57.967 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T22:27:57.968 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T22:27:57.968 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T22:27:58.382 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:58.383 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:58.384 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:58.388 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:58.390 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:58.392 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:58.393 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:58.395 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:27:58.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:27:58 smithi105 ceph-mon[23707]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:27:58.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:27:58 smithi118 ceph-mon[25717]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:00.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:00 smithi105 ceph-mon[23707]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:00 smithi118 ceph-mon[25717]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:02 smithi118 ceph-mon[25717]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:03.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:02 smithi105 ceph-mon[23707]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:04.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:04 smithi105 ceph-mon[23707]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:04.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:04 smithi118 ceph-mon[25717]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:06.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:06 smithi105 ceph-mon[23707]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:06 smithi118 ceph-mon[25717]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:08.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:08 smithi105 ceph-mon[23707]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:08 smithi118 ceph-mon[25717]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:09.631 INFO:teuthology.orchestra.run.smithi105.stdout:137438953496 2024-09-13T22:28:09.632 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T22:28:10.551 INFO:teuthology.orchestra.run.smithi105.stdout:206158430223 2024-09-13T22:28:10.552 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T22:28:10.810 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:10 smithi105 ceph-mon[23707]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:10 smithi118 ceph-mon[25717]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:11.594 INFO:teuthology.orchestra.run.smithi105.stdout:38654705710 2024-09-13T22:28:11.594 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T22:28:12.086 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:12.541 INFO:teuthology.orchestra.run.smithi105.stdout:240518168585 2024-09-13T22:28:12.541 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T22:28:12.861 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:12 smithi105 ceph-mon[23707]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:12 smithi118 ceph-mon[25717]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:13.941 INFO:teuthology.orchestra.run.smithi105.stdout:60129542184 2024-09-13T22:28:13.942 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T22:28:14.383 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:14.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:14 smithi105 ceph-mon[23707]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:14.827 INFO:teuthology.orchestra.run.smithi105.stdout:115964117022 2024-09-13T22:28:14.828 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T22:28:14.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:14 smithi118 ceph-mon[25717]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:15.010 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:15.885 INFO:teuthology.orchestra.run.smithi105.stdout:171798691860 2024-09-13T22:28:15.885 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T22:28:15.886 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:16.313 INFO:teuthology.orchestra.run.smithi105.stdout:81604378659 2024-09-13T22:28:16.313 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T22:28:16.959 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:16 smithi105 ceph-mon[23707]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:16 smithi118 ceph-mon[25717]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:17.280 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:17.281 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:18.837 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:18 smithi105 ceph-mon[23707]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:19.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:18 smithi118 ceph-mon[25717]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:20.669 INFO:teuthology.orchestra.run.smithi105.stdout:240518168587 2024-09-13T22:28:20.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:20 smithi105 ceph-mon[23707]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:20.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:20 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1103997536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T22:28:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:20 smithi118 ceph-mon[25717]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:20 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/1103997536' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T22:28:21.439 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168585 got 240518168587 for osd.7 2024-09-13T22:28:21.439 DEBUG:teuthology.parallel:result is None 2024-09-13T22:28:22.396 INFO:teuthology.orchestra.run.smithi105.stdout:115964117023 2024-09-13T22:28:23.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:22 smithi118 ceph-mon[25717]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:23.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:22 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/910132481' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T22:28:23.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:22 smithi105 ceph-mon[23707]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:23.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:22 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/910132481' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T22:28:23.858 INFO:teuthology.orchestra.run.smithi105.stdout:81604378661 2024-09-13T22:28:24.268 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:24.588 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:24 smithi105 ceph-mon[23707]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:24.588 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:24 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2295968099' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T22:28:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:24 smithi118 ceph-mon[25717]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:24 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2295968099' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T22:28:25.772 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:28:25.772 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:28:25.794 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117022 got 115964117023 for osd.3 2024-09-13T22:28:25.794 DEBUG:teuthology.parallel:result is None 2024-09-13T22:28:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:28:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:28:26.763 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:26 smithi105 ceph-mon[23707]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:26.914 INFO:teuthology.orchestra.run.smithi105.stdout:206158430226 2024-09-13T22:28:27.001 INFO:teuthology.orchestra.run.smithi105.stdout:137438953500 2024-09-13T22:28:27.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:26 smithi118 ceph-mon[25717]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:27.514 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430223 got 206158430226 for osd.6 2024-09-13T22:28:27.515 DEBUG:teuthology.parallel:result is None 2024-09-13T22:28:27.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:27 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/347382285' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T22:28:27.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:27 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2022015612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T22:28:28.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:27 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/347382285' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T22:28:28.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:27 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2022015612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T22:28:28.247 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378659 got 81604378661 for osd.2 2024-09-13T22:28:28.248 DEBUG:teuthology.parallel:result is None 2024-09-13T22:28:28.675 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953496 got 137438953500 for osd.4 2024-09-13T22:28:28.675 DEBUG:teuthology.parallel:result is None 2024-09-13T22:28:28.838 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:28.959 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:28 smithi105 ceph-mon[23707]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:29.053 INFO:teuthology.orchestra.run.smithi105.stdout:60129542188 2024-09-13T22:28:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:28 smithi118 ceph-mon[25717]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:29.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:29 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1022818344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T22:28:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:29 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/1022818344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T22:28:30.460 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542188 for osd.1 2024-09-13T22:28:30.460 DEBUG:teuthology.parallel:result is None 2024-09-13T22:28:30.990 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:30 smithi105 ceph-mon[23707]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:30 smithi118 ceph-mon[25717]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:31.500 INFO:teuthology.orchestra.run.smithi105.stdout:38654705714 2024-09-13T22:28:31.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:31 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/196083795' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T22:28:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:31 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/196083795' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T22:28:32.120 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705710 got 38654705714 for osd.0 2024-09-13T22:28:32.120 DEBUG:teuthology.parallel:result is None 2024-09-13T22:28:32.505 INFO:teuthology.orchestra.run.smithi105.stdout:171798691863 2024-09-13T22:28:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:32 smithi118 ceph-mon[25717]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:32 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/954699495' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T22:28:33.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:32 smithi105 ceph-mon[23707]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:33.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:32 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/954699495' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T22:28:33.203 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691860 got 171798691863 for osd.5 2024-09-13T22:28:33.204 DEBUG:teuthology.parallel:result is None 2024-09-13T22:28:33.204 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T22:28:33.204 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T22:28:33.410 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:34.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:34 smithi105 ceph-mon[23707]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:34.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:34 smithi118 ceph-mon[25717]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:35.130 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:28:35.132 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-13T22:28:35.682 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-13T22:28:33.622906+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48812,"kb_used_data":6380,"kb_used_omap":0,"kb_used_meta":42368,"kb_avail":749912404,"statfs":{"total":767960285184,"available":767910301696,"internally_reserved":0,"allocated":6533120,"data_stored":3572009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43384832},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002110"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-13T22:27:40.626736+0000","last_change":"2024-09-13T22:27:40.626736+0000","last_active":"2024-09-13T22:27:40.626736+0000","last_peered":"2024-09-13T22:27:40.626736+0000","last_clean":"2024-09-13T22:27:40.626736+0000","last_became_active":"2024-09-13T22:27:40.626316+0000","last_became_peered":"2024-09-13T22:27:40.626316+0000","last_unstale":"2024-09-13T22:27:40.626736+0000","last_undegraded":"2024-09-13T22:27:40.626736+0000","last_fullsized":"2024-09-13T22:27:40.626736+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T22:25:36.332292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T22:25:36.332292+0000","last_clean_scrub_stamp":"2024-09-13T22:25:36.332292+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:06.757170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66800000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59599999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55000000000000004}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38300000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41799999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49199999999999999}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6120,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739032,"statfs":{"total":95995035648,"available":95988768768,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.436},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.57199999999999995}]},{"osd":1,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.505}]},{"osd":2,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.47699999999999998}]},{"osd":3,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.298,"15min":1.298},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":15.811,"5min":15.811,"15min":15.811},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":15.788,"5min":15.788,"15min":15.788},"last":0.622}]},{"osd":4,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.60099999999999998}]},{"osd":5,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66500000000000004}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:28:13 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.33800000000000002,"15min":0.32300000000000001},"min":{"1min":0.247,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.49199999999999999,"5min":0.625,"15min":0.625},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.35299999999999998,"15min":0.32700000000000001},"min":{"1min":0.28199999999999997,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.51200000000000001,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.38200000000000001}]},{"osd":2,"last update":"Fri Sep 13 22:27:37 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.371,"15min":0.36399999999999999},"min":{"1min":0.28599999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.432,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.41599999999999998}]},{"osd":3,"last update":"Fri Sep 13 22:28:09 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.41499999999999998,"15min":0.41199999999999998},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.435},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.36899999999999999}]},{"osd":4,"last update":"Fri Sep 13 22:28:26 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.48299999999999998,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.60599999999999998,"15min":0.60199999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.749}]},{"osd":5,"last update":"Fri Sep 13 22:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.76600000000000001}]},{"osd":6,"last update":"Fri Sep 13 22:28:13 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.67500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78700000000000003}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 22:28:12 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.34499999999999997,"15min":0.314},"min":{"1min":0.28699999999999998,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.59599999999999997,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.377,"5min":0.34399999999999997,"15min":0.317},"min":{"1min":0.29699999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.54900000000000004,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.28399999999999997}]},{"osd":2,"last update":"Fri Sep 13 22:27:39 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.39100000000000001,"15min":0.379},"min":{"1min":0.29099999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40000000000000002,"15min":0.39700000000000002},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.56899999999999995,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.34999999999999998}]},{"osd":3,"last update":"Fri Sep 13 22:28:06 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41399999999999998,"15min":0.42199999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.50600000000000001,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47999999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.34300000000000003}]},{"osd":4,"last update":"Fri Sep 13 22:28:27 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.54200000000000004,"15min":0.52800000000000002},"min":{"1min":0.46600000000000003,"5min":0.436,"15min":0.436},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.53600000000000003,"15min":0.52300000000000002},"min":{"1min":0.41399999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54100000000000004}]},{"osd":5,"last update":"Fri Sep 13 22:27:48 2024","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":10.297000000000001,"5min":10.297000000000001,"15min":10.297000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.52300000000000002}]},{"osd":6,"last update":"Fri Sep 13 22:28:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.53300000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6244,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:27:36 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.34399999999999997,"15min":0.34000000000000002},"min":{"1min":0.22,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.51900000000000002,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.317},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.34599999999999997,"15min":0.33800000000000002},"min":{"1min":0.27900000000000003,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.27100000000000002}]},{"osd":1,"last update":"Fri Sep 13 22:27:36 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.34499999999999997},"min":{"1min":0.252,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.32200000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.42199999999999999}]},{"osd":3,"last update":"Fri Sep 13 22:28:05 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.41799999999999998,"15min":0.41699999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.45300000000000001,"15min":0.45100000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.39700000000000002}]},{"osd":4,"last update":"Fri Sep 13 22:28:26 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.56999999999999995,"15min":0.56499999999999995},"min":{"1min":0.36799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80300000000000005,"5min":0.879,"15min":0.879},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.86699999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.40799999999999997}]},{"osd":5,"last update":"Fri Sep 13 22:27:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.34899999999999998}]},{"osd":6,"last update":"Fri Sep 13 22:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0429999999999999,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.54000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:28:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.35699999999999998,"15min":0.35299999999999998},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.19700000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.41899999999999998,"15min":0.41799999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.318}]},{"osd":1,"last update":"Fri Sep 13 22:28:04 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.42799999999999999,"15min":0.42199999999999999},"min":{"1min":0.28499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.41499999999999998,"15min":0.41299999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.27800000000000002}]},{"osd":2,"last update":"Fri Sep 13 22:28:04 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.439,"15min":0.443},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.245}]},{"osd":4,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.378,"5min":0.37,"15min":0.37},"max":{"1min":0.77600000000000002,"5min":1.071,"15min":1.071},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77700000000000002,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.40799999999999997}]},{"osd":5,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.36199999999999999}]},{"osd":6,"last update":"Fri Sep 13 22:28:15 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.45700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42199999999999999}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.54300000000000004,"15min":0.55000000000000004},"min":{"1min":0.371,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70499999999999996,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.39100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.66600000000000004,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.55600000000000005}]},{"osd":1,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.52900000000000003,"15min":0.53800000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.623,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.55200000000000005,"15min":0.55700000000000005},"min":{"1min":0.39500000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.67900000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.53100000000000003}]},{"osd":2,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71099999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69899999999999995,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.51800000000000002}]},{"osd":3,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.57499999999999996,"15min":0.58399999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.68999999999999995,"5min":1.014,"15min":1.014},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53200000000000003,"15min":0.53500000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.66800000000000004,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.46400000000000002}]},{"osd":5,"last update":"Fri Sep 13 22:27:52 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.379},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.41799999999999998}]},{"osd":6,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.39900000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.54200000000000004}]},{"osd":1,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":3.5539999999999998,"5min":3.5539999999999998,"15min":3.5539999999999998},"last":0.51200000000000001}]},{"osd":2,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.6190000000000002,"5min":3.6190000000000002,"15min":3.6190000000000002},"last":0.5},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.63200000000000001}]},{"osd":3,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":3.585,"5min":3.585,"15min":3.585},"last":0.52700000000000002}]},{"osd":4,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.45300000000000001}]},{"osd":6,"last update":"Fri Sep 13 22:28:15 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.48199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T22:28:35.684 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T22:28:35.890 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:37.023 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:36 smithi105 ceph-mon[23707]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:28:37.023 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:36 smithi105 ceph-mon[23707]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:37.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:36 smithi118 ceph-mon[25717]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:28:37.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:36 smithi118 ceph-mon[25717]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:37.660 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:28:37.661 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-13T22:28:38.400 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":204,"stamp":"2024-09-13T22:28:37.623466+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48812,"kb_used_data":6380,"kb_used_omap":0,"kb_used_meta":42368,"kb_avail":749912404,"statfs":{"total":767960285184,"available":767910301696,"internally_reserved":0,"allocated":6533120,"data_stored":3572009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43384832},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002102"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-13T22:27:40.626736+0000","last_change":"2024-09-13T22:27:40.626736+0000","last_active":"2024-09-13T22:27:40.626736+0000","last_peered":"2024-09-13T22:27:40.626736+0000","last_clean":"2024-09-13T22:27:40.626736+0000","last_became_active":"2024-09-13T22:27:40.626316+0000","last_became_peered":"2024-09-13T22:27:40.626316+0000","last_unstale":"2024-09-13T22:27:40.626736+0000","last_undegraded":"2024-09-13T22:27:40.626736+0000","last_fullsized":"2024-09-13T22:27:40.626736+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T22:25:36.332292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T22:25:36.332292+0000","last_clean_scrub_stamp":"2024-09-13T22:25:36.332292+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:06.757170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168591,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.501}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73399999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.755}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6120,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739032,"statfs":{"total":95995035648,"available":95988768768,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.436},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.57199999999999995}]},{"osd":1,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.505}]},{"osd":2,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.47699999999999998}]},{"osd":3,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":1.298,"15min":1.298},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":15.811,"5min":15.811,"15min":15.811},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":15.788,"5min":15.788,"15min":15.788},"last":0.622}]},{"osd":4,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.60099999999999998}]},{"osd":5,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66500000000000004}]}]},{"osd":1,"up_from":14,"seq":60129542190,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:28:13 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.33800000000000002,"15min":0.32300000000000001},"min":{"1min":0.247,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.49199999999999999,"5min":0.625,"15min":0.625},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.35299999999999998,"15min":0.32700000000000001},"min":{"1min":0.28199999999999997,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.51200000000000001,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.35699999999999998}]},{"osd":2,"last update":"Fri Sep 13 22:27:37 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.371,"15min":0.36399999999999999},"min":{"1min":0.28599999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.25700000000000001}]},{"osd":3,"last update":"Fri Sep 13 22:28:09 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.41499999999999998,"15min":0.41199999999999998},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.307},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.34100000000000003}]},{"osd":4,"last update":"Fri Sep 13 22:28:26 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.48299999999999998,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.60599999999999998,"15min":0.60199999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.58399999999999996}]},{"osd":5,"last update":"Fri Sep 13 22:27:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.628},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.56299999999999994}]},{"osd":6,"last update":"Fri Sep 13 22:28:13 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.60299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 22:28:12 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.34499999999999997,"15min":0.314},"min":{"1min":0.28699999999999998,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.59599999999999997,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.377,"5min":0.34399999999999997,"15min":0.317},"min":{"1min":0.29699999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.54900000000000004,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.28399999999999997}]},{"osd":2,"last update":"Fri Sep 13 22:27:39 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.39100000000000001,"15min":0.379},"min":{"1min":0.29099999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40000000000000002,"15min":0.39700000000000002},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.56899999999999995,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.34999999999999998}]},{"osd":3,"last update":"Fri Sep 13 22:28:06 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41399999999999998,"15min":0.42199999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.50600000000000001,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.47199999999999998,"15min":0.47999999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.34300000000000003}]},{"osd":4,"last update":"Fri Sep 13 22:28:27 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.54200000000000004,"15min":0.52800000000000002},"min":{"1min":0.46600000000000003,"5min":0.436,"15min":0.436},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.53600000000000003,"15min":0.52300000000000002},"min":{"1min":0.41399999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54100000000000004}]},{"osd":5,"last update":"Fri Sep 13 22:27:48 2024","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":10.297000000000001,"5min":10.297000000000001,"15min":10.297000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.52300000000000002}]},{"osd":6,"last update":"Fri Sep 13 22:28:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.53300000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6244,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:27:36 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.34399999999999997,"15min":0.34000000000000002},"min":{"1min":0.22,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.51900000000000002,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.34599999999999997,"15min":0.33800000000000002},"min":{"1min":0.27900000000000003,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.32100000000000001}]},{"osd":1,"last update":"Fri Sep 13 22:27:36 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.34499999999999997},"min":{"1min":0.252,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.434,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.32200000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.34799999999999998}]},{"osd":3,"last update":"Fri Sep 13 22:28:05 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.41799999999999998,"15min":0.41699999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.45300000000000001,"15min":0.45100000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.378}]},{"osd":4,"last update":"Fri Sep 13 22:28:26 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.56999999999999995,"15min":0.56499999999999995},"min":{"1min":0.36799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80300000000000005,"5min":0.879,"15min":0.879},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.86699999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.73299999999999998}]},{"osd":5,"last update":"Fri Sep 13 22:27:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.58699999999999997}]},{"osd":6,"last update":"Fri Sep 13 22:28:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0429999999999999,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.66400000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.75}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:28:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.35699999999999998,"15min":0.35299999999999998},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.19700000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.41899999999999998,"15min":0.41799999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.318}]},{"osd":1,"last update":"Fri Sep 13 22:28:04 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.42799999999999999,"15min":0.42199999999999999},"min":{"1min":0.28499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.41499999999999998,"15min":0.41299999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.27800000000000002}]},{"osd":2,"last update":"Fri Sep 13 22:28:04 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.439,"15min":0.443},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.245}]},{"osd":4,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.378,"5min":0.37,"15min":0.37},"max":{"1min":0.77600000000000002,"5min":1.071,"15min":1.071},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.77700000000000002,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.40799999999999997}]},{"osd":5,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.36199999999999999}]},{"osd":6,"last update":"Fri Sep 13 22:28:15 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.45700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42199999999999999}]}]},{"osd":4,"up_from":32,"seq":137438953502,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.54300000000000004,"15min":0.55000000000000004},"min":{"1min":0.371,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70499999999999996,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.39100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.66600000000000004,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.41699999999999998}]},{"osd":1,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.52900000000000003,"15min":0.53800000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.623,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.55200000000000005,"15min":0.55700000000000005},"min":{"1min":0.39500000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.67900000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.49399999999999999}]},{"osd":2,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.71099999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.69899999999999995,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.35399999999999998}]},{"osd":3,"last update":"Fri Sep 13 22:28:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.57499999999999996,"15min":0.58399999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.68999999999999995,"5min":1.014,"15min":1.014},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53200000000000003,"15min":0.53500000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.66800000000000004,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.433}]},{"osd":5,"last update":"Fri Sep 13 22:27:52 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.48199999999999998}]},{"osd":6,"last update":"Fri Sep 13 22:28:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.53200000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":5,"up_from":40,"seq":171798691865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.46400000000000002}]},{"osd":1,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":3.5539999999999998,"5min":3.5539999999999998,"15min":3.5539999999999998},"last":0.40699999999999997}]},{"osd":2,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.6190000000000002,"5min":3.6190000000000002,"15min":3.6190000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.48099999999999998}]},{"osd":3,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.435},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":3.585,"5min":3.585,"15min":3.585},"last":0.51700000000000002}]},{"osd":4,"last update":"Fri Sep 13 22:27:49 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.52500000000000002}]},{"osd":6,"last update":"Fri Sep 13 22:28:15 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.57999999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T22:28:38.401 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T22:28:38.402 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T22:28:38.402 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T22:28:38.402 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T22:28:38.610 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/mon.smithi105/config 2024-09-13T22:28:38.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:38 smithi118 ceph-mon[25717]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:38.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:38 smithi118 ceph-mon[25717]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:28:38.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:28:38.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:28:38.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:28:38.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:28:38.989 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:38 smithi105 ceph-mon[23707]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:38.989 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:38 smithi105 ceph-mon[23707]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:28:38.989 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:28:38.989 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:28:38.989 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:28:38.989 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:28:40.389 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:28:40.389 INFO:teuthology.orchestra.run.smithi105.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T22:28:40.999 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:40 smithi105 ceph-mon[23707]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:40.999 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:40 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2661385822' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T22:28:41.022 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T22:28:41.023 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T22:28:41.023 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T22:28:41.032 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:28:41.032 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-13T22:28:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:40 smithi118 ceph-mon[25717]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:40 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2661385822' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T22:28:42.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:42 smithi105 ceph-mon[23707]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:43.031 INFO:teuthology.orchestra.run.smithi105.stdout:Backend: cephadm 2024-09-13T22:28:43.031 INFO:teuthology.orchestra.run.smithi105.stdout:Available: Yes 2024-09-13T22:28:43.031 INFO:teuthology.orchestra.run.smithi105.stdout:Paused: No 2024-09-13T22:28:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:42 smithi118 ceph-mon[25717]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:43.661 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-13T22:28:44.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:44 smithi105 ceph-mon[23707]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:44.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:44 smithi105 ceph-mon[23707]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:44 smithi118 ceph-mon[25717]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:44 smithi118 ceph-mon[25717]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:45.629 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:28:45.629 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c baade4bebf2e 2024-09-13T22:28:45.629 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:28:45.629 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (5m) 70s ago 5m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:28:45.629 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (4m) 2m ago 5m 41.1M - 8.3.5 dad864ee21e9 20a1ee977603 2024-09-13T22:28:45.629 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 3fdf94a6360e 2024-09-13T22:28:45.629 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283 running (5m) 70s ago 5m 410M - 17.2.0 e1d6a67b021e 70a20d7579fb 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (7m) 2m ago 7m 44.4M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (5m) 70s ago 5m 39.1M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (5m) 2m ago 5m 15.5M - 1dbe0e931976 71bdc25e0af6 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (4m) 70s ago 4m 17.0M - 1dbe0e931976 66d8f794097b 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (4m) 2m ago 4m 36.1M 3216M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (3m) 2m ago 3m 37.5M 3216M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (3m) 2m ago 3m 31.0M 3216M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (2m) 2m ago 2m 27.6M 3216M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (2m) 70s ago 2m 34.1M 3984M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (2m) 70s ago 2m 32.8M 3984M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (100s) 70s ago 99s 31.8M 3984M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (75s) 70s ago 75s 10.9M 3984M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:28:45.630 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (4m) 2m ago 4m 54.2M - 514e6a882f6e da708a26e310 2024-09-13T22:28:46.248 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-13T22:28:46.798 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:46 smithi105 ceph-mon[23707]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:46.798 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:46 smithi105 ceph-mon[23707]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:47.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:46 smithi118 ceph-mon[25717]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:47.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:46 smithi118 ceph-mon[25717]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:48.154 INFO:teuthology.orchestra.run.smithi105.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-13T22:28:48.155 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-13T22:28:48.155 INFO:teuthology.orchestra.run.smithi105.stdout:crash 2/2 2m ago 6m * 2024-09-13T22:28:48.155 INFO:teuthology.orchestra.run.smithi105.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-13T22:28:48.155 INFO:teuthology.orchestra.run.smithi105.stdout:mgr 2/2 2m ago 6m count:2 2024-09-13T22:28:48.155 INFO:teuthology.orchestra.run.smithi105.stdout:mon 2/2 2m ago 5m smithi105:172.21.15.105=smithi105;smithi118:172.21.15.118=smithi118;count:2 2024-09-13T22:28:48.155 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-13T22:28:48.155 INFO:teuthology.orchestra.run.smithi105.stdout:osd 8 2m ago - 2024-09-13T22:28:48.155 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-13T22:28:48.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:48 smithi105 ceph-mon[23707]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:48.836 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-13T22:28:49.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:48 smithi118 ceph-mon[25717]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:49.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:49 smithi105 ceph-mon[23707]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:50.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:49 smithi118 ceph-mon[25717]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:50.738 INFO:teuthology.orchestra.run.smithi105.stdout:HOST ADDR LABELS STATUS 2024-09-13T22:28:50.738 INFO:teuthology.orchestra.run.smithi105.stdout:smithi105 172.21.15.105 2024-09-13T22:28:50.738 INFO:teuthology.orchestra.run.smithi105.stdout:smithi118 172.21.15.118 2024-09-13T22:28:50.738 INFO:teuthology.orchestra.run.smithi105.stdout:2 hosts in cluster 2024-09-13T22:28:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:50 smithi118 ceph-mon[25717]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:51.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:50 smithi105 ceph-mon[23707]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:51.450 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-13T22:28:52.003 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:51 smithi105 ceph-mon[23707]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:52.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:51 smithi118 ceph-mon[25717]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:52.756 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:52 smithi105 ceph-mon[23707]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:52 smithi118 ceph-mon[25717]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:53.590 INFO:teuthology.orchestra.run.smithi105.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-13T22:28:53.590 INFO:teuthology.orchestra.run.smithi105.stdout:smithi105 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400TR400BGN 400G LVM detected, locked 2024-09-13T22:28:53.590 INFO:teuthology.orchestra.run.smithi105.stdout:smithi118 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB747500JS480DGN 480G LVM detected, locked 2024-09-13T22:28:54.353 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T22:28:54.366 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:28:54.368 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-13T22:28:54.393 INFO:teuthology.orchestra.run.smithi105.stderr:+ systemctl stop nfs-server 2024-09-13T22:28:54.402 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi118.front.sepia.ceph.com 2024-09-13T22:28:54.402 DEBUG:teuthology.orchestra.run.smithi118:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-13T22:28:54.433 INFO:teuthology.orchestra.run.smithi118.stderr:+ systemctl stop nfs-server 2024-09-13T22:28:54.443 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T22:28:54.451 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:28:54.451 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-13T22:28:54.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:54 smithi105 ceph-mon[23707]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:54.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:54 smithi105 ceph-mon[23707]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:54 smithi118 ceph-mon[25717]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:54 smithi118 ceph-mon[25717]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:56 smithi118 ceph-mon[25717]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:56 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-13T22:28:57.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:56 smithi105 ceph-mon[23707]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:57.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:56 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-13T22:28:58.026 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:57 smithi105 ceph-mon[23707]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:58.026 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:57 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-13T22:28:58.026 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:57 smithi105 ceph-mon[23707]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T22:28:58.026 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:57 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-13T22:28:58.026 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:57 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105[23702]: 2024-09-13T22:28:57.696+0000 7f4082ff7700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T22:28:58.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:57 smithi118 ceph-mon[25717]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:28:58.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:57 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-13T22:28:58.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:57 smithi118 ceph-mon[25717]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T22:28:58.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:57 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-13T22:28:58.330 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-13T22:28:58.339 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-13T22:28:58.339 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T22:28:58.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: pgmap v215: 33 pgs: 5 creating+peering, 1 active+clean, 27 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:58.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-13T22:28:58.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T22:28:58.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-13T22:28:58.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: fsmap foofs:0 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: Saving service mds.foofs spec with placement count:2 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi105.ddoxtb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi105.ddoxtb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:28:58.871 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:58 smithi105 ceph-mon[23707]: Deploying daemon mds.foofs.smithi105.ddoxtb on smithi105 2024-09-13T22:28:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: pgmap v215: 33 pgs: 5 creating+peering, 1 active+clean, 27 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:28:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-13T22:28:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T22:28:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-13T22:28:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: fsmap foofs:0 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: Saving service mds.foofs spec with placement count:2 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:28:59.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:28:59.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi105.ddoxtb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:28:59.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi105.ddoxtb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T22:28:59.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:28:59.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:58 smithi118 ceph-mon[25717]: Deploying daemon mds.foofs.smithi105.ddoxtb on smithi105 2024-09-13T22:29:00.054 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:28:59 smithi105 ceph-mon[23707]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T22:29:00.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:28:59 smithi118 ceph-mon[25717]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T22:29:00.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:00 smithi105 ceph-mon[23707]: pgmap v219: 65 pgs: 1 creating+activating, 14 creating+peering, 14 active+clean, 36 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:29:00.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:00 smithi105 ceph-mon[23707]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T22:29:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:00 smithi118 ceph-mon[25717]: pgmap v219: 65 pgs: 1 creating+activating, 14 creating+peering, 14 active+clean, 36 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:29:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:00 smithi118 ceph-mon[25717]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T22:29:02.152 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:29:02.152 INFO:teuthology.orchestra.run.smithi105.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T22:22:24.273291Z", "last_refresh": "2024-09-13T22:26:09.624922Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T22:22:18.040786Z", "last_refresh": "2024-09-13T22:26:09.624986Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T22:22:20.947057Z", "last_refresh": "2024-09-13T22:26:09.625048Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T22:28:57.717970Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T22:28:57.713669Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T22:22:15.908528Z", "last_refresh": "2024-09-13T22:26:09.624850Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi105:172.21.15.105=smithi105", "smithi118:172.21.15.118=smithi118"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T22:23:08.132071Z", "last_refresh": "2024-09-13T22:26:09.624699Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T22:22:22.269727Z", "last_refresh": "2024-09-13T22:26:09.625123Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T22:26:09.625238Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T22:22:19.465524Z", "last_refresh": "2024-09-13T22:26:09.625180Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T22:29:02.837 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: pgmap v221: 65 pgs: 1 creating+activating, 14 creating+peering, 27 active+clean, 23 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:29:02.837 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: mds.? [v2:172.21.15.105:6834/1075044016,v1:172.21.15.105:6835/1075044016] up:boot 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: daemon mds.foofs.smithi105.ddoxtb assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: Cluster is now healthy 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: fsmap foofs:0 1 up:standby 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:creating} 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.izbofj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.izbofj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: daemon mds.foofs.smithi105.ddoxtb is now active in filesystem foofs as rank 0 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:02.838 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:02 smithi105 ceph-mon[23707]: Deploying daemon mds.foofs.smithi118.izbofj on smithi118 2024-09-13T22:29:02.860 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-13T22:29:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: pgmap v221: 65 pgs: 1 creating+activating, 14 creating+peering, 27 active+clean, 23 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:29:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: mds.? [v2:172.21.15.105:6834/1075044016,v1:172.21.15.105:6835/1075044016] up:boot 2024-09-13T22:29:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: daemon mds.foofs.smithi105.ddoxtb assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-13T22:29:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T22:29:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T22:29:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: Cluster is now healthy 2024-09-13T22:29:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: fsmap foofs:0 1 up:standby 2024-09-13T22:29:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:29:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:creating} 2024-09-13T22:29:03.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:03.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.izbofj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:29:03.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.izbofj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T22:29:03.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: daemon mds.foofs.smithi105.ddoxtb is now active in filesystem foofs as rank 0 2024-09-13T22:29:03.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:03.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:02 smithi118 ceph-mon[25717]: Deploying daemon mds.foofs.smithi118.izbofj on smithi118 2024-09-13T22:29:03.839 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:03 smithi118 ceph-mon[25717]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:29:03.840 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:03 smithi118 ceph-mon[25717]: mds.? [v2:172.21.15.105:6834/1075044016,v1:172.21.15.105:6835/1075044016] up:active 2024-09-13T22:29:03.840 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:03 smithi118 ceph-mon[25717]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 2024-09-13T22:29:03.861 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T22:29:04.119 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:03 smithi105 ceph-mon[23707]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:29:04.119 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:03 smithi105 ceph-mon[23707]: mds.? [v2:172.21.15.105:6834/1075044016,v1:172.21.15.105:6835/1075044016] up:active 2024-09-13T22:29:04.119 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:03 smithi105 ceph-mon[23707]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 2024-09-13T22:29:04.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:04 smithi118 ceph-mon[25717]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 2 op/s 2024-09-13T22:29:04.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:04.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:04.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:04.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:04 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:05.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:04 smithi105 ceph-mon[23707]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 2 op/s 2024-09-13T22:29:05.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:05.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:05.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:05.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:04 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:05.809 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:29:05.809 INFO:teuthology.orchestra.run.smithi105.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T22:22:24.273291Z", "last_refresh": "2024-09-13T22:26:09.624922Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T22:22:18.040786Z", "last_refresh": "2024-09-13T22:26:09.624986Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T22:22:20.947057Z", "last_refresh": "2024-09-13T22:26:09.625048Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T22:28:57.717970Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T22:28:57.713669Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T22:22:15.908528Z", "last_refresh": "2024-09-13T22:26:09.624850Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi105:172.21.15.105=smithi105", "smithi118:172.21.15.118=smithi118"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T22:23:08.132071Z", "last_refresh": "2024-09-13T22:26:09.624699Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T22:22:22.269727Z", "last_refresh": "2024-09-13T22:26:09.625123Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T22:26:09.625238Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T22:22:19.465524Z", "last_refresh": "2024-09-13T22:26:09.625180Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T22:29:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:05 smithi118 ceph-mon[25717]: mds.? [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] up:boot 2024-09-13T22:29:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:05 smithi118 ceph-mon[25717]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:29:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:29:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:05 smithi118 ceph-mon[25717]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:29:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:05 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:05 smithi105 ceph-mon[23707]: mds.? [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] up:boot 2024-09-13T22:29:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:05 smithi105 ceph-mon[23707]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:29:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:29:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:05 smithi105 ceph-mon[23707]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:29:06.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:05 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:06.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:06 smithi105 ceph-mon[23707]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 4 op/s 2024-09-13T22:29:06.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:06 smithi105 ceph-mon[23707]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:29:06.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:06 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:06.978 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-13T22:29:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:06 smithi118 ceph-mon[25717]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 4 op/s 2024-09-13T22:29:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:06 smithi118 ceph-mon[25717]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:29:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:06 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:07.980 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T22:29:08.794 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:08 smithi105 ceph-mon[23707]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-09-13T22:29:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:08 smithi118 ceph-mon[25717]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-09-13T22:29:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:10 smithi105 ceph-mon[23707]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T22:29:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:10 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:10 smithi118 ceph-mon[25717]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T22:29:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:10 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:11.740 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:29:11.740 INFO:teuthology.orchestra.run.smithi105.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T22:22:24.273291Z", "last_refresh": "2024-09-13T22:29:11.711964Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T22:22:18.040786Z", "last_refresh": "2024-09-13T22:29:05.968784Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T22:22:20.947057Z", "last_refresh": "2024-09-13T22:29:11.712107Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T22:28:57.717970Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T22:28:57.713669Z", "last_refresh": "2024-09-13T22:29:05.969386Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T22:22:15.908528Z", "last_refresh": "2024-09-13T22:29:05.968948Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi105:172.21.15.105=smithi105", "smithi118:172.21.15.118=smithi118"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T22:23:08.132071Z", "last_refresh": "2024-09-13T22:29:05.969017Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T22:22:22.269727Z", "last_refresh": "2024-09-13T22:29:05.969096Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T22:29:05.969154Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T22:22:19.465524Z", "last_refresh": "2024-09-13T22:29:11.712223Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T22:29:12.418 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-13T22:29:12.418 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T22:29:12.427 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:29:12.427 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-13T22:29:13.015 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:12 smithi105 ceph-mon[23707]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T22:29:13.015 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:13.015 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:13.015 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:13.015 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:13.015 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:13.015 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:12 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:13.016 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:12 smithi105 ceph-mon[23707]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:29:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:12 smithi118 ceph-mon[25717]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T22:29:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:12 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:12 smithi118 ceph-mon[25717]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:29:14.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:14 smithi105 ceph-mon[23707]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T22:29:14.895 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:14 smithi118 ceph-mon[25717]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T22:29:15.684 INFO:teuthology.orchestra.run.smithi105.stdout:NFS Cluster Created Successfully 2024-09-13T22:29:15.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:15 smithi105 ceph-mon[23707]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:15.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-13T22:29:15.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-13T22:29:15.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:15 smithi105 ceph-mon[23707]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T22:29:15.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:15 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-13T22:29:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:15 smithi118 ceph-mon[25717]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-13T22:29:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-13T22:29:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:15 smithi118 ceph-mon[25717]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T22:29:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:15 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-13T22:29:16.405 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-13T22:29:16.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: pgmap v229: 97 pgs: 6 creating+peering, 26 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T22:29:16.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-13T22:29:16.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T22:29:16.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: Saving service nfs.foo spec with placement count:2 2024-09-13T22:29:16.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:16.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:16.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: Creating key for client.nfs.foo.0.0.smithi105.xxfcdx 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:29:16.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T22:29:16.850 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:16 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:16.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: pgmap v229: 97 pgs: 6 creating+peering, 26 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T22:29:16.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-13T22:29:16.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T22:29:16.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: Saving service nfs.foo spec with placement count:2 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: Creating key for client.nfs.foo.0.0.smithi105.xxfcdx 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T22:29:16.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:16.939 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T22:29:16.939 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T22:29:16.939 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:29:16.939 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T22:29:16.939 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:16 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:18.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:17 smithi105 ceph-mon[23707]: Rados config object exists: conf-nfs.foo 2024-09-13T22:29:18.076 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:17 smithi105 ceph-mon[23707]: Creating key for client.nfs.foo.0.0.smithi105.xxfcdx-rgw 2024-09-13T22:29:18.076 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:17 smithi105 ceph-mon[23707]: Deploying daemon nfs.foo.0.0.smithi105.xxfcdx on smithi105 2024-09-13T22:29:18.076 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:17 smithi105 ceph-mon[23707]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T22:29:18.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:17 smithi118 ceph-mon[25717]: Rados config object exists: conf-nfs.foo 2024-09-13T22:29:18.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:17 smithi118 ceph-mon[25717]: Creating key for client.nfs.foo.0.0.smithi105.xxfcdx-rgw 2024-09-13T22:29:18.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:17 smithi118 ceph-mon[25717]: Deploying daemon nfs.foo.0.0.smithi105.xxfcdx on smithi105 2024-09-13T22:29:18.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:17 smithi118 ceph-mon[25717]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T22:29:18.851 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:18 smithi105 ceph-mon[23707]: pgmap v232: 97 pgs: 6 creating+peering, 24 unknown, 67 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:29:18.851 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:18 smithi105 ceph-mon[23707]: mgrmap e20: smithi105.swmmea(active, since 5m), standbys: smithi118.rbhtgz 2024-09-13T22:29:19.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:18 smithi118 ceph-mon[25717]: pgmap v232: 97 pgs: 6 creating+peering, 24 unknown, 67 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T22:29:19.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:18 smithi118 ceph-mon[25717]: mgrmap e20: smithi105.swmmea(active, since 5m), standbys: smithi118.rbhtgz 2024-09-13T22:29:19.636 INFO:teuthology.orchestra.run.smithi105.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-13T22:29:19.637 INFO:teuthology.orchestra.run.smithi105.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-13T22:29:19.637 INFO:teuthology.orchestra.run.smithi105.stderr:Error EINVAL: invalid command 2024-09-13T22:29:19.985 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:29:19.985 INFO:teuthology.orchestra.run.smithi105.stdout: "bind": "/fake", 2024-09-13T22:29:19.985 INFO:teuthology.orchestra.run.smithi105.stdout: "fs": "foofs", 2024-09-13T22:29:19.985 INFO:teuthology.orchestra.run.smithi105.stdout: "path": "/", 2024-09-13T22:29:19.986 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster": "foo", 2024-09-13T22:29:19.986 INFO:teuthology.orchestra.run.smithi105.stdout: "mode": "RW" 2024-09-13T22:29:19.986 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:29:20.829 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:20 smithi105 ceph-mon[23707]: pgmap v233: 97 pgs: 6 creating+peering, 91 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T22:29:20.829 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-13T22:29:20.829 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-13T22:29:20.829 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:20.829 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T22:29:20.829 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T22:29:20.829 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T22:29:20.829 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T22:29:20.830 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:20 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:20.852 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-13T22:29:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:20 smithi118 ceph-mon[25717]: pgmap v233: 97 pgs: 6 creating+peering, 91 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T22:29:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-13T22:29:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-13T22:29:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T22:29:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T22:29:21.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T22:29:21.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T22:29:21.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:20 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:22.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:21 smithi105 ceph-mon[23707]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:22.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:21 smithi105 ceph-mon[23707]: Creating key for client.nfs.foo.1.0.smithi118.chlqvt 2024-09-13T22:29:22.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:21 smithi105 ceph-mon[23707]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T22:29:22.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:21 smithi118 ceph-mon[25717]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:22.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:21 smithi118 ceph-mon[25717]: Creating key for client.nfs.foo.1.0.smithi118.chlqvt 2024-09-13T22:29:22.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:21 smithi118 ceph-mon[25717]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T22:29:22.949 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:22 smithi105 ceph-mon[23707]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 146 B/s rd, 293 B/s wr, 0 op/s 2024-09-13T22:29:23.037 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:22 smithi118 ceph-mon[25717]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 146 B/s rd, 293 B/s wr, 0 op/s 2024-09-13T22:29:24.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:23 smithi118 ceph-mon[25717]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:24.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:23 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T22:29:24.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:23 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T22:29:24.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:23 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:29:24.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:23 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T22:29:24.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:23 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:24.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:23 smithi105 ceph-mon[23707]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:24.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:23 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T22:29:24.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:23 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T22:29:24.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:23 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:29:24.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:23 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T22:29:24.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:23 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:24.823 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:24 smithi118 ceph-mon[25717]: Rados config object exists: conf-nfs.foo 2024-09-13T22:29:24.823 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:24 smithi118 ceph-mon[25717]: Creating key for client.nfs.foo.1.0.smithi118.chlqvt-rgw 2024-09-13T22:29:24.823 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:24 smithi118 ceph-mon[25717]: Deploying daemon nfs.foo.1.0.smithi118.chlqvt on smithi118 2024-09-13T22:29:24.823 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:24 smithi118 ceph-mon[25717]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-13T22:29:25.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:24 smithi105 ceph-mon[23707]: Rados config object exists: conf-nfs.foo 2024-09-13T22:29:25.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:24 smithi105 ceph-mon[23707]: Creating key for client.nfs.foo.1.0.smithi118.chlqvt-rgw 2024-09-13T22:29:25.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:24 smithi105 ceph-mon[23707]: Deploying daemon nfs.foo.1.0.smithi118.chlqvt on smithi118 2024-09-13T22:29:25.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:24 smithi105 ceph-mon[23707]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-13T22:29:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:25.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:25.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-13T22:29:25.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-13T22:29:25.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T22:29:25.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-13T22:29:25.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:29:25.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:29:25.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:26.093 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:29:26.094 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:26.945 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:26 smithi118 ceph-mon[25717]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:26.946 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:26 smithi118 ceph-mon[25717]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T22:29:26.946 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:26 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-13T22:29:26.946 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:26 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-09-13T22:29:26.946 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:26 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-13T22:29:26.946 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:26 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-13T22:29:26.946 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:26 smithi118 ceph-mon[25717]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T22:29:27.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:26 smithi105 ceph-mon[23707]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:27.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:26 smithi105 ceph-mon[23707]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T22:29:27.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-13T22:29:27.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-09-13T22:29:27.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-13T22:29:27.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-13T22:29:27.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:26 smithi105 ceph-mon[23707]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T22:29:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:27 smithi105 ceph-mon[23707]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:27 smithi105 ceph-mon[23707]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T22:29:28.072 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:27 smithi118 ceph-mon[25717]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:28.072 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:27 smithi118 ceph-mon[25717]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T22:29:28.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:28 smithi105 ceph-mon[23707]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T22:29:28.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:28 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:28 smithi118 ceph-mon[25717]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-13T22:29:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:28 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:30.036 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:29 smithi105 ceph-mon[23707]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:29 smithi118 ceph-mon[25717]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:30.594 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-09-13T22:29:30.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:30.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: pgmap v240: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-13T22:29:30.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:30.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:30.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:30.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:30.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:30.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:30.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:30.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:30.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:30 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: pgmap v240: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-13T22:29:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:31.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:31.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:31.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:30 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:31.392 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T22:29:31.401 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:29:31.402 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-13T22:29:31.428 INFO:teuthology.orchestra.run.smithi105.stderr:+ mkdir /mnt/foo 2024-09-13T22:29:31.436 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-13T22:29:31.500 INFO:teuthology.orchestra.run.smithi105.stderr:++ hostname 2024-09-13T22:29:31.501 INFO:teuthology.orchestra.run.smithi105.stderr:+ mount -t nfs smithi105:/fake /mnt/foo -o sync 2024-09-13T22:29:31.910 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:31 smithi105 ceph-mon[23707]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:31.947 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-13T22:29:32.011 INFO:teuthology.orchestra.run.smithi105.stderr:+ echo test 2024-09-13T22:29:32.021 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-13T22:29:32.086 INFO:teuthology.orchestra.run.smithi105.stderr:+ sync 2024-09-13T22:29:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:31 smithi118 ceph-mon[25717]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:32.278 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T22:29:32.288 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T22:29:32.288 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T22:29:32.289 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T22:29:32.289 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:29:32.290 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T22:29:32.290 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T22:29:32.290 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-13T22:29:32.291 INFO:teuthology.task.exec:Executing custom commands... 2024-09-13T22:29:32.291 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:29:32.291 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-13T22:29:32.373 INFO:teuthology.orchestra.run.smithi105.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-13T22:29:32.374 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:29:32.374 INFO:teuthology.orchestra.run.smithi105.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-13T22:29:32.374 INFO:teuthology.orchestra.run.smithi105.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-13T22:29:32.475 INFO:teuthology.orchestra.run.smithi105.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-13T22:29:32.475 INFO:teuthology.orchestra.run.smithi105.stdout:releasing clients 2024-09-13T22:29:32.788 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:32 smithi105 ceph-mon[23707]: pgmap v241: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.0 KiB/s wr, 5 op/s; 69 B/s, 0 objects/s recovering 2024-09-13T22:29:33.096 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:32 smithi118 ceph-mon[25717]: pgmap v241: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.0 KiB/s wr, 5 op/s; 69 B/s, 0 objects/s recovering 2024-09-13T22:29:33.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 120 27.12 MB/sec warmup 1 sec latency 31.306 ms 2024-09-13T22:29:34.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:34 smithi105 ceph-mon[23707]: pgmap v242: 97 pgs: 97 active+clean; 3.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 349 KiB/s wr, 32 op/s; 69 B/s, 0 objects/s recovering 2024-09-13T22:29:34.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 246 23.32 MB/sec warmup 2 sec latency 57.520 ms 2024-09-13T22:29:34.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:34 smithi118 ceph-mon[25717]: pgmap v242: 97 pgs: 97 active+clean; 3.1 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 349 KiB/s wr, 32 op/s; 69 B/s, 0 objects/s recovering 2024-09-13T22:29:35.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 385 22.78 MB/sec warmup 3 sec latency 31.358 ms 2024-09-13T22:29:35.608 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T22:29:36.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 521 22.59 MB/sec warmup 4 sec latency 25.139 ms 2024-09-13T22:29:36.785 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:36 smithi105 ceph-mon[23707]: pgmap v243: 97 pgs: 97 active+clean; 12 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.3 MiB/s wr, 46 op/s; 60 B/s, 0 objects/s recovering 2024-09-13T22:29:37.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:36 smithi118 ceph-mon[25717]: pgmap v243: 97 pgs: 97 active+clean; 12 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.3 MiB/s wr, 46 op/s; 60 B/s, 0 objects/s recovering 2024-09-13T22:29:37.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 698 21.23 MB/sec warmup 5 sec latency 39.276 ms 2024-09-13T22:29:38.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 1079 19.10 MB/sec warmup 6 sec latency 32.666 ms 2024-09-13T22:29:38.942 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:38 smithi105 ceph-mon[23707]: pgmap v244: 97 pgs: 97 active+clean; 43 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.2 MiB/s wr, 99 op/s; 54 B/s, 0 objects/s recovering 2024-09-13T22:29:39.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:38 smithi118 ceph-mon[25717]: pgmap v244: 97 pgs: 97 active+clean; 43 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.2 MiB/s wr, 99 op/s; 54 B/s, 0 objects/s recovering 2024-09-13T22:29:39.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 2054 19.01 MB/sec warmup 7 sec latency 18.100 ms 2024-09-13T22:29:39.713 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T22:29:40.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 2725 19.28 MB/sec warmup 8 sec latency 83.177 ms 2024-09-13T22:29:40.707 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:40 smithi105 ceph-mon[23707]: pgmap v245: 97 pgs: 97 active+clean; 102 MiB data, 151 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 8.8 MiB/s wr, 374 op/s; 46 B/s, 0 objects/s recovering 2024-09-13T22:29:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:40 smithi118 ceph-mon[25717]: pgmap v245: 97 pgs: 97 active+clean; 102 MiB data, 151 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 8.8 MiB/s wr, 374 op/s; 46 B/s, 0 objects/s recovering 2024-09-13T22:29:41.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 3378 20.16 MB/sec warmup 9 sec latency 32.371 ms 2024-09-13T22:29:42.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 3829 19.81 MB/sec warmup 10 sec latency 36.542 ms 2024-09-13T22:29:42.740 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:42 smithi105 ceph-mon[23707]: pgmap v246: 97 pgs: 97 active+clean; 116 MiB data, 178 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 10 MiB/s wr, 511 op/s 2024-09-13T22:29:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:42 smithi118 ceph-mon[25717]: pgmap v246: 97 pgs: 97 active+clean; 116 MiB data, 178 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 10 MiB/s wr, 511 op/s 2024-09-13T22:29:43.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 4384 18.47 MB/sec warmup 11 sec latency 46.946 ms 2024-09-13T22:29:44.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 4971 18.57 MB/sec warmup 12 sec latency 32.400 ms 2024-09-13T22:29:44.506 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:44 smithi105 ceph-mon[23707]: pgmap v247: 97 pgs: 97 active+clean; 128 MiB data, 202 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 13 MiB/s wr, 704 op/s 2024-09-13T22:29:44.624 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-13T22:29:44.846 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:44 smithi118 ceph-mon[25717]: pgmap v247: 97 pgs: 97 active+clean; 128 MiB data, 202 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 13 MiB/s wr, 704 op/s 2024-09-13T22:29:45.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 5700 17.69 MB/sec warmup 13 sec latency 32.645 ms 2024-09-13T22:29:46.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 6153 17.32 MB/sec warmup 14 sec latency 36.723 ms 2024-09-13T22:29:46.809 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:46 smithi105 ceph-mon[23707]: pgmap v248: 97 pgs: 97 active+clean; 131 MiB data, 238 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 13 MiB/s wr, 712 op/s 2024-09-13T22:29:47.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:46 smithi118 ceph-mon[25717]: pgmap v248: 97 pgs: 97 active+clean; 131 MiB data, 238 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 13 MiB/s wr, 712 op/s 2024-09-13T22:29:47.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 6764 17.88 MB/sec warmup 15 sec latency 71.243 ms 2024-09-13T22:29:47.721 INFO:teuthology.orchestra.run.smithi105.stderr:module 'nfs' is already enabled 2024-09-13T22:29:47.907 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:47 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2920560194' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-13T22:29:48.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:47 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2920560194' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-13T22:29:48.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 7281 18.25 MB/sec warmup 16 sec latency 57.343 ms 2024-09-13T22:29:48.865 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:48 smithi105 ceph-mon[23707]: pgmap v249: 97 pgs: 97 active+clean; 130 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 13 MiB/s wr, 746 op/s 2024-09-13T22:29:48.865 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:48 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2920560194' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-13T22:29:48.865 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:48 smithi105 ceph-mon[23707]: mgrmap e21: smithi105.swmmea(active, since 6m), standbys: smithi118.rbhtgz 2024-09-13T22:29:48.892 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T22:29:49.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:48 smithi118 ceph-mon[25717]: pgmap v249: 97 pgs: 97 active+clean; 130 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 13 MiB/s wr, 746 op/s 2024-09-13T22:29:49.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:48 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2920560194' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-13T22:29:49.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:48 smithi118 ceph-mon[25717]: mgrmap e21: smithi105.swmmea(active, since 6m), standbys: smithi118.rbhtgz 2024-09-13T22:29:49.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 7496 17.39 MB/sec warmup 17 sec latency 140.762 ms 2024-09-13T22:29:50.475 INFO:teuthology.orchestra.run.smithi105.stdout: 5 7875 16.55 MB/sec warmup 18 sec latency 54.695 ms 2024-09-13T22:29:50.889 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:50 smithi105 ceph-mon[23707]: pgmap v250: 97 pgs: 97 active+clean; 135 MiB data, 302 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 13 MiB/s wr, 942 op/s 2024-09-13T22:29:51.096 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:50 smithi118 ceph-mon[25717]: pgmap v250: 97 pgs: 97 active+clean; 135 MiB data, 302 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 13 MiB/s wr, 942 op/s 2024-09-13T22:29:51.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 8233 16.06 MB/sec warmup 19 sec latency 30.411 ms 2024-09-13T22:29:51.998 INFO:teuthology.orchestra.run.smithi105.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:29:52.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 8606 15.94 MB/sec warmup 20 sec latency 45.606 ms 2024-09-13T22:29:52.945 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:52 smithi105 ceph-mon[23707]: pgmap v251: 97 pgs: 97 active+clean; 147 MiB data, 327 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.0 MiB/s wr, 834 op/s 2024-09-13T22:29:52.946 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:52 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:52.946 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:52 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:52.946 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:52 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:52.946 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:52 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:52.946 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:52 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:52.946 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:52 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:52 smithi118 ceph-mon[25717]: pgmap v251: 97 pgs: 97 active+clean; 147 MiB data, 327 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.0 MiB/s wr, 834 op/s 2024-09-13T22:29:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:52 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:52 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:29:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:52 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:29:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:52 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:29:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:52 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:52 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:29:53.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 9043 15.40 MB/sec warmup 21 sec latency 91.961 ms 2024-09-13T22:29:53.822 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T22:29:53.823 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:29:53.823 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-13T22:29:53.974 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:53 smithi105 ceph-mon[23707]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:53.974 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:53 smithi105 ceph-mon[23707]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:29:53.974 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:53 smithi105 ceph-mon[23707]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:29:54.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:53 smithi118 ceph-mon[25717]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:54.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:53 smithi118 ceph-mon[25717]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:29:54.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:53 smithi118 ceph-mon[25717]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 2024-09-13T22:29:54.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 9461 15.00 MB/sec warmup 22 sec latency 60.137 ms 2024-09-13T22:29:54.753 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:54 smithi105 ceph-mon[23707]: pgmap v252: 97 pgs: 97 active+clean; 143 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.1 MiB/s wr, 737 op/s 2024-09-13T22:29:55.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:54 smithi118 ceph-mon[25717]: pgmap v252: 97 pgs: 97 active+clean; 143 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.1 MiB/s wr, 737 op/s 2024-09-13T22:29:55.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 9620 14.54 MB/sec warmup 23 sec latency 63.878 ms 2024-09-13T22:29:56.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 10218 15.10 MB/sec warmup 24 sec latency 52.870 ms 2024-09-13T22:29:56.805 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:56 smithi105 ceph-mon[23707]: pgmap v253: 97 pgs: 97 active+clean; 139 MiB data, 420 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 6.0 MiB/s wr, 560 op/s 2024-09-13T22:29:56.861 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:29:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:56 smithi118 ceph-mon[25717]: pgmap v253: 97 pgs: 97 active+clean; 139 MiB data, 420 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 6.0 MiB/s wr, 560 op/s 2024-09-13T22:29:57.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 10799 15.47 MB/sec warmup 25 sec latency 30.095 ms 2024-09-13T22:29:57.813 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:57 smithi105 ceph-mon[23707]: from='client.24371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:57.831 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (5m) 27s ago 6m 22.2M - ba2b418f427c baade4bebf2e 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (6m) 27s ago 6m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (6m) 29s ago 6m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (5m) 27s ago 6m 48.9M - 8.3.5 dad864ee21e9 20a1ee977603 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (56s) 27s ago 56s 13.5M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (54s) 29s ago 53s 12.1M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:9283 running (8m) 27s ago 8m 460M - 17.2.0 e1d6a67b021e 3fdf94a6360e 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283 running (6m) 29s ago 6m 411M - 17.2.0 e1d6a67b021e 70a20d7579fb 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (8m) 27s ago 8m 48.1M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (6m) 29s ago 6m 34.4M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:29:57.832 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (38s) 27s ago 37s 49.0M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (32s) 29s ago 32s 46.7M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (6m) 27s ago 6m 17.7M - 1dbe0e931976 71bdc25e0af6 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (6m) 29s ago 6m 17.2M - 1dbe0e931976 66d8f794097b 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (5m) 27s ago 5m 45.1M 3216M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (5m) 27s ago 5m 45.8M 3216M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (4m) 27s ago 4m 40.1M 3216M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (4m) 27s ago 4m 42.4M 3216M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (3m) 29s ago 3m 40.0M 3984M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (3m) 29s ago 3m 38.8M 3984M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (2m) 29s ago 2m 39.9M 3984M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (2m) 29s ago 2m 43.5M 3984M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:29:57.833 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (5m) 27s ago 5m 62.9M - 514e6a882f6e da708a26e310 2024-09-13T22:29:58.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:57 smithi118 ceph-mon[25717]: from='client.24371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:29:58.359 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:29:58.360 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:29:58.360 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:29:58.360 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:29:58.360 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T22:29:58.360 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:29:58.360 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:29:58.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 11109 15.06 MB/sec warmup 26 sec latency 42.659 ms 2024-09-13T22:29:58.803 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:29:58.804 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:29:58.804 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:29:58.804 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [], 2024-09-13T22:29:58.804 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "", 2024-09-13T22:29:58.804 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 image" 2024-09-13T22:29:58.804 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:29:58.827 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:58 smithi105 ceph-mon[23707]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:58.827 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:58 smithi105 ceph-mon[23707]: pgmap v254: 97 pgs: 97 active+clean; 143 MiB data, 427 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.4 MiB/s wr, 660 op/s 2024-09-13T22:29:58.827 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:58 smithi105 ceph-mon[23707]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:58.827 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:58 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2198746010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:29:58.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:58 smithi118 ceph-mon[25717]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:58.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:58 smithi118 ceph-mon[25717]: pgmap v254: 97 pgs: 97 active+clean; 143 MiB data, 427 MiB used, 715 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.4 MiB/s wr, 660 op/s 2024-09-13T22:29:58.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:58 smithi118 ceph-mon[25717]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:58.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:58 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2198746010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:29:59.321 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:29:59.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 11456 14.59 MB/sec warmup 27 sec latency 42.080 ms 2024-09-13T22:29:59.836 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:59 smithi105 ceph-mon[23707]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:29:59.836 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:29:59 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2769699876' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:30:00.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:59 smithi118 ceph-mon[25717]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:30:00.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:29:59 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2769699876' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:30:00.476 INFO:teuthology.orchestra.run.smithi105.stdout: 5 11741 14.27 MB/sec warmup 28 sec latency 51.670 ms 2024-09-13T22:30:00.860 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:00 smithi105 ceph-mon[23707]: pgmap v255: 97 pgs: 97 active+clean; 152 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 747 op/s 2024-09-13T22:30:00.860 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:00 smithi105 ceph-mon[23707]: overall HEALTH_OK 2024-09-13T22:30:01.096 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:00 smithi118 ceph-mon[25717]: pgmap v255: 97 pgs: 97 active+clean; 152 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 747 op/s 2024-09-13T22:30:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:00 smithi118 ceph-mon[25717]: overall HEALTH_OK 2024-09-13T22:30:01.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 12109 14.19 MB/sec warmup 29 sec latency 48.642 ms 2024-09-13T22:30:02.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 12567 13.97 MB/sec warmup 30 sec latency 58.573 ms 2024-09-13T22:30:02.899 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:02 smithi105 ceph-mon[23707]: pgmap v256: 97 pgs: 97 active+clean; 156 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.9 MiB/s wr, 569 op/s 2024-09-13T22:30:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:02 smithi118 ceph-mon[25717]: pgmap v256: 97 pgs: 97 active+clean; 156 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.9 MiB/s wr, 569 op/s 2024-09-13T22:30:03.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 13020 13.75 MB/sec warmup 31 sec latency 48.209 ms 2024-09-13T22:30:04.431 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:04 smithi105 ceph-mon[23707]: pgmap v257: 97 pgs: 97 active+clean; 152 MiB data, 465 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 506 op/s 2024-09-13T22:30:04.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 13573 14.09 MB/sec warmup 32 sec latency 34.612 ms 2024-09-13T22:30:04.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:04 smithi118 ceph-mon[25717]: pgmap v257: 97 pgs: 97 active+clean; 152 MiB data, 465 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 506 op/s 2024-09-13T22:30:05.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 14135 14.34 MB/sec warmup 33 sec latency 30.924 ms 2024-09-13T22:30:06.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 14524 14.31 MB/sec warmup 34 sec latency 52.277 ms 2024-09-13T22:30:06.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:06 smithi105 ceph-mon[23707]: pgmap v258: 97 pgs: 97 active+clean; 150 MiB data, 491 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.9 MiB/s wr, 484 op/s 2024-09-13T22:30:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:06 smithi118 ceph-mon[25717]: pgmap v258: 97 pgs: 97 active+clean; 150 MiB data, 491 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.9 MiB/s wr, 484 op/s 2024-09-13T22:30:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 15167 14.09 MB/sec warmup 35 sec latency 41.847 ms 2024-09-13T22:30:08.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 15510 13.99 MB/sec warmup 36 sec latency 33.118 ms 2024-09-13T22:30:08.743 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:08 smithi105 ceph-mon[23707]: pgmap v259: 97 pgs: 97 active+clean; 165 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.0 MiB/s wr, 656 op/s 2024-09-13T22:30:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:08 smithi118 ceph-mon[25717]: pgmap v259: 97 pgs: 97 active+clean; 165 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.0 MiB/s wr, 656 op/s 2024-09-13T22:30:09.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 15960 13.89 MB/sec warmup 37 sec latency 76.753 ms 2024-09-13T22:30:10.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 16456 13.67 MB/sec warmup 38 sec latency 38.680 ms 2024-09-13T22:30:10.772 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:10 smithi105 ceph-mon[23707]: pgmap v260: 97 pgs: 97 active+clean; 172 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.3 MiB/s wr, 650 op/s 2024-09-13T22:30:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:10 smithi118 ceph-mon[25717]: pgmap v260: 97 pgs: 97 active+clean; 172 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.3 MiB/s wr, 650 op/s 2024-09-13T22:30:11.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 16896 13.77 MB/sec warmup 39 sec latency 53.647 ms 2024-09-13T22:30:12.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 17562 14.03 MB/sec warmup 40 sec latency 50.484 ms 2024-09-13T22:30:12.808 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:12 smithi105 ceph-mon[23707]: pgmap v261: 97 pgs: 97 active+clean; 174 MiB data, 583 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 642 op/s 2024-09-13T22:30:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:12 smithi118 ceph-mon[25717]: pgmap v261: 97 pgs: 97 active+clean; 174 MiB data, 583 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 642 op/s 2024-09-13T22:30:13.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 17963 14.15 MB/sec warmup 41 sec latency 44.362 ms 2024-09-13T22:30:14.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 18243 13.88 MB/sec warmup 42 sec latency 52.853 ms 2024-09-13T22:30:14.502 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:14 smithi105 ceph-mon[23707]: pgmap v262: 97 pgs: 97 active+clean; 172 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.8 MiB/s wr, 639 op/s 2024-09-13T22:30:14.846 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:14 smithi118 ceph-mon[25717]: pgmap v262: 97 pgs: 97 active+clean; 172 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.8 MiB/s wr, 639 op/s 2024-09-13T22:30:15.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 18695 13.66 MB/sec warmup 43 sec latency 64.823 ms 2024-09-13T22:30:16.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 19322 13.77 MB/sec warmup 44 sec latency 48.250 ms 2024-09-13T22:30:16.864 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:16 smithi105 ceph-mon[23707]: pgmap v263: 97 pgs: 97 active+clean; 177 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 633 op/s 2024-09-13T22:30:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:16 smithi118 ceph-mon[25717]: pgmap v263: 97 pgs: 97 active+clean; 177 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 633 op/s 2024-09-13T22:30:17.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 20069 13.70 MB/sec warmup 45 sec latency 27.842 ms 2024-09-13T22:30:18.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 20629 13.88 MB/sec warmup 46 sec latency 26.368 ms 2024-09-13T22:30:18.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:18 smithi105 ceph-mon[23707]: pgmap v264: 97 pgs: 97 active+clean; 184 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 8.2 MiB/s wr, 827 op/s 2024-09-13T22:30:19.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:18 smithi118 ceph-mon[25717]: pgmap v264: 97 pgs: 97 active+clean; 184 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 8.2 MiB/s wr, 827 op/s 2024-09-13T22:30:19.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 21230 14.06 MB/sec warmup 47 sec latency 31.788 ms 2024-09-13T22:30:20.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 21593 14.09 MB/sec warmup 48 sec latency 44.550 ms 2024-09-13T22:30:20.934 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:20 smithi105 ceph-mon[23707]: pgmap v265: 97 pgs: 97 active+clean; 181 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.7 MiB/s wr, 679 op/s 2024-09-13T22:30:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:20 smithi118 ceph-mon[25717]: pgmap v265: 97 pgs: 97 active+clean; 181 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.7 MiB/s wr, 679 op/s 2024-09-13T22:30:21.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 21873 13.86 MB/sec warmup 49 sec latency 61.846 ms 2024-09-13T22:30:22.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 22228 13.66 MB/sec warmup 50 sec latency 62.197 ms 2024-09-13T22:30:22.967 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:22 smithi105 ceph-mon[23707]: pgmap v266: 97 pgs: 97 active+clean; 187 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 743 op/s 2024-09-13T22:30:23.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:22 smithi118 ceph-mon[25717]: pgmap v266: 97 pgs: 97 active+clean; 187 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 743 op/s 2024-09-13T22:30:23.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 22541 13.55 MB/sec warmup 51 sec latency 46.192 ms 2024-09-13T22:30:24.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 23001 13.50 MB/sec warmup 52 sec latency 45.400 ms 2024-09-13T22:30:24.493 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:24 smithi105 ceph-mon[23707]: pgmap v267: 97 pgs: 97 active+clean; 195 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.5 MiB/s wr, 668 op/s 2024-09-13T22:30:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:24 smithi118 ceph-mon[25717]: pgmap v267: 97 pgs: 97 active+clean; 195 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.5 MiB/s wr, 668 op/s 2024-09-13T22:30:25.477 INFO:teuthology.orchestra.run.smithi105.stdout: 5 23580 13.42 MB/sec warmup 53 sec latency 34.999 ms 2024-09-13T22:30:25.757 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:30:25.757 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:25 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:30:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:30:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:25 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:30:26.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 24005 13.49 MB/sec warmup 54 sec latency 32.245 ms 2024-09-13T22:30:26.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:26 smithi105 ceph-mon[23707]: pgmap v268: 97 pgs: 97 active+clean; 194 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 622 op/s 2024-09-13T22:30:27.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:26 smithi118 ceph-mon[25717]: pgmap v268: 97 pgs: 97 active+clean; 194 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 622 op/s 2024-09-13T22:30:27.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 24643 13.67 MB/sec warmup 55 sec latency 37.407 ms 2024-09-13T22:30:28.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 25105 13.78 MB/sec warmup 56 sec latency 36.029 ms 2024-09-13T22:30:28.823 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:28 smithi105 ceph-mon[23707]: pgmap v269: 97 pgs: 97 active+clean; 196 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 657 op/s 2024-09-13T22:30:28.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:28 smithi118 ceph-mon[25717]: pgmap v269: 97 pgs: 97 active+clean; 196 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 657 op/s 2024-09-13T22:30:29.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 25431 13.59 MB/sec warmup 57 sec latency 55.052 ms 2024-09-13T22:30:29.764 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:30:30.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 25999 13.55 MB/sec warmup 58 sec latency 32.542 ms 2024-09-13T22:30:30.712 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (6m) 60s ago 7m 22.2M - ba2b418f427c baade4bebf2e 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (7m) 60s ago 7m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (6m) 62s ago 6m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (5m) 60s ago 7m 48.9M - 8.3.5 dad864ee21e9 20a1ee977603 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (89s) 60s ago 89s 13.5M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (87s) 62s ago 86s 12.1M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:9283 running (8m) 60s ago 8m 460M - 17.2.0 e1d6a67b021e 3fdf94a6360e 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283 running (6m) 62s ago 6m 411M - 17.2.0 e1d6a67b021e 70a20d7579fb 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (8m) 60s ago 8m 48.1M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (6m) 62s ago 6m 34.4M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:30:30.713 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (71s) 60s ago 70s 49.0M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (65s) 62s ago 65s 46.7M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (7m) 60s ago 7m 17.7M - 1dbe0e931976 71bdc25e0af6 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (6m) 62s ago 6m 17.2M - 1dbe0e931976 66d8f794097b 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (6m) 60s ago 5m 45.1M 3216M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (5m) 60s ago 5m 45.8M 3216M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (5m) 60s ago 5m 40.1M 3216M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (4m) 60s ago 4m 42.4M 3216M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (4m) 62s ago 4m 40.0M 3984M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (3m) 62s ago 3m 38.8M 3984M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (3m) 62s ago 3m 39.9M 3984M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (3m) 62s ago 3m 43.5M 3984M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:30:30.714 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (6m) 60s ago 6m 62.9M - 514e6a882f6e da708a26e310 2024-09-13T22:30:30.855 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:30 smithi105 ceph-mon[23707]: pgmap v270: 97 pgs: 97 active+clean; 201 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.6 MiB/s wr, 586 op/s 2024-09-13T22:30:30.855 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:30 smithi105 ceph-mon[23707]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:30:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:30 smithi118 ceph-mon[25717]: pgmap v270: 97 pgs: 97 active+clean; 201 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.6 MiB/s wr, 586 op/s 2024-09-13T22:30:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:30 smithi118 ceph-mon[25717]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:30:31.239 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:30:31.239 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:30:31.239 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:30:31.239 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:30:31.239 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:30:31.240 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:30:31.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 26791 13.60 MB/sec warmup 59 sec latency 29.235 ms 2024-09-13T22:30:31.717 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:30:31.717 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50", 2024-09-13T22:30:31.717 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:30:31.717 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [], 2024-09-13T22:30:31.717 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "", 2024-09-13T22:30:31.717 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:ecab7be58963a5bd03910cd73eef66405d7fdd50 image" 2024-09-13T22:30:31.718 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:30:31.862 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:31 smithi105 ceph-mon[23707]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:30:31.862 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:31 smithi105 ceph-mon[23707]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:30:31.862 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:31 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2237783512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:30:31.943 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:31 smithi118 ceph-mon[25717]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:30:31.943 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:31 smithi118 ceph-mon[25717]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:30:31.943 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:31 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2237783512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:30:32.298 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:30:32.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 27544 13.69 MB/sec warmup 60 sec latency 24.764 ms 2024-09-13T22:30:33.124 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:32 smithi105 ceph-mon[23707]: pgmap v271: 97 pgs: 97 active+clean; 208 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 671 op/s 2024-09-13T22:30:33.124 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:32 smithi105 ceph-mon[23707]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:30:33.124 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:30:33.124 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:32 smithi105 ceph-mon[23707]: Upgrade: Target is version 19.1.1-324-gecab7be5 (unknown) 2024-09-13T22:30:33.124 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:32 smithi105 ceph-mon[23707]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:30:33.124 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:30:33.125 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:32 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:30:33.125 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:32 smithi105 ceph-mon[23707]: Upgrade: Need to upgrade myself (mgr.smithi105.swmmea) 2024-09-13T22:30:33.125 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:32 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1271348674' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:30:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:32 smithi118 ceph-mon[25717]: pgmap v271: 97 pgs: 97 active+clean; 208 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 671 op/s 2024-09-13T22:30:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:32 smithi118 ceph-mon[25717]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:30:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:32 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:30:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:32 smithi118 ceph-mon[25717]: Upgrade: Target is version 19.1.1-324-gecab7be5 (unknown) 2024-09-13T22:30:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:32 smithi118 ceph-mon[25717]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808'] 2024-09-13T22:30:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:32 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:30:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:32 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:30:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:32 smithi118 ceph-mon[25717]: Upgrade: Need to upgrade myself (mgr.smithi105.swmmea) 2024-09-13T22:30:33.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:32 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/1271348674' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:30:33.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 28276 13.95 MB/sec warmup 61 sec latency 27.411 ms 2024-09-13T22:30:34.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:33 smithi105 ceph-mon[23707]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 on smithi118 2024-09-13T22:30:34.346 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:33 smithi118 ceph-mon[25717]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 on smithi118 2024-09-13T22:30:34.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 28620 13.97 MB/sec warmup 62 sec latency 39.305 ms 2024-09-13T22:30:34.906 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:34 smithi105 ceph-mon[23707]: pgmap v272: 97 pgs: 97 active+clean; 210 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.4 MiB/s wr, 647 op/s 2024-09-13T22:30:35.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:34 smithi118 ceph-mon[25717]: pgmap v272: 97 pgs: 97 active+clean; 210 MiB data, 876 MiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.4 MiB/s wr, 647 op/s 2024-09-13T22:30:35.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 28870 13.80 MB/sec warmup 63 sec latency 65.366 ms 2024-09-13T22:30:36.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 29178 13.62 MB/sec warmup 64 sec latency 48.965 ms 2024-09-13T22:30:36.959 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:36 smithi105 ceph-mon[23707]: pgmap v273: 97 pgs: 97 active+clean; 211 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 628 op/s 2024-09-13T22:30:37.346 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:36 smithi118 ceph-mon[25717]: pgmap v273: 97 pgs: 97 active+clean; 211 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 628 op/s 2024-09-13T22:30:37.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 29502 13.50 MB/sec warmup 65 sec latency 39.365 ms 2024-09-13T22:30:38.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 29946 13.48 MB/sec warmup 66 sec latency 51.088 ms 2024-09-13T22:30:39.022 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:38 smithi105 ceph-mon[23707]: pgmap v274: 97 pgs: 97 active+clean; 217 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.1 MiB/s wr, 792 op/s 2024-09-13T22:30:39.346 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:38 smithi118 ceph-mon[25717]: pgmap v274: 97 pgs: 97 active+clean; 217 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.1 MiB/s wr, 792 op/s 2024-09-13T22:30:39.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 30437 13.43 MB/sec warmup 67 sec latency 48.470 ms 2024-09-13T22:30:40.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 30808 13.31 MB/sec warmup 68 sec latency 53.035 ms 2024-09-13T22:30:41.100 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:40 smithi105 ceph-mon[23707]: pgmap v275: 97 pgs: 97 active+clean; 221 MiB data, 943 MiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.6 MiB/s wr, 743 op/s 2024-09-13T22:30:41.346 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:40 smithi118 ceph-mon[25717]: pgmap v275: 97 pgs: 97 active+clean; 221 MiB data, 943 MiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.6 MiB/s wr, 743 op/s 2024-09-13T22:30:41.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 31158 13.35 MB/sec warmup 69 sec latency 54.550 ms 2024-09-13T22:30:42.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 31573 13.37 MB/sec warmup 70 sec latency 66.795 ms 2024-09-13T22:30:43.171 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:42 smithi105 ceph-mon[23707]: pgmap v276: 97 pgs: 97 active+clean; 216 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 625 op/s 2024-09-13T22:30:43.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:42 smithi118 ceph-mon[25717]: pgmap v276: 97 pgs: 97 active+clean; 216 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 625 op/s 2024-09-13T22:30:43.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 31931 13.41 MB/sec warmup 71 sec latency 52.481 ms 2024-09-13T22:30:44.447 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:44 smithi105 ceph-mon[23707]: pgmap v277: 97 pgs: 97 active+clean; 224 MiB data, 994 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.1 MiB/s wr, 675 op/s 2024-09-13T22:30:44.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 32183 13.36 MB/sec warmup 72 sec latency 50.660 ms 2024-09-13T22:30:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:44 smithi118 ceph-mon[25717]: pgmap v277: 97 pgs: 97 active+clean; 224 MiB data, 994 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.1 MiB/s wr, 675 op/s 2024-09-13T22:30:45.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 32452 13.22 MB/sec warmup 73 sec latency 112.848 ms 2024-09-13T22:30:46.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 32903 13.12 MB/sec warmup 74 sec latency 40.648 ms 2024-09-13T22:30:46.792 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:46 smithi105 ceph-mon[23707]: pgmap v278: 97 pgs: 97 active+clean; 225 MiB data, 1014 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.7 MiB/s wr, 534 op/s 2024-09-13T22:30:47.096 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:46 smithi118 ceph-mon[25717]: pgmap v278: 97 pgs: 97 active+clean; 225 MiB data, 1014 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.7 MiB/s wr, 534 op/s 2024-09-13T22:30:47.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 33340 13.13 MB/sec warmup 75 sec latency 38.807 ms 2024-09-13T22:30:48.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 33740 13.02 MB/sec warmup 76 sec latency 55.884 ms 2024-09-13T22:30:48.860 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:48 smithi105 ceph-mon[23707]: pgmap v279: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.4 MiB/s wr, 614 op/s 2024-09-13T22:30:49.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:48 smithi118 ceph-mon[25717]: pgmap v279: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.4 MiB/s wr, 614 op/s 2024-09-13T22:30:49.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 34177 12.96 MB/sec warmup 77 sec latency 46.808 ms 2024-09-13T22:30:50.479 INFO:teuthology.orchestra.run.smithi105.stdout: 5 34595 12.96 MB/sec warmup 78 sec latency 54.102 ms 2024-09-13T22:30:50.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:50 smithi105 ceph-mon[23707]: pgmap v280: 97 pgs: 97 active+clean; 235 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 556 op/s 2024-09-13T22:30:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:50 smithi118 ceph-mon[25717]: pgmap v280: 97 pgs: 97 active+clean; 235 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 556 op/s 2024-09-13T22:30:51.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 35371 13.19 MB/sec warmup 79 sec latency 43.215 ms 2024-09-13T22:30:52.479 INFO:teuthology.orchestra.run.smithi105.stdout: 5 35901 13.26 MB/sec warmup 80 sec latency 32.919 ms 2024-09-13T22:30:52.917 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:52 smithi105 ceph-mon[23707]: pgmap v281: 97 pgs: 97 active+clean; 234 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 489 op/s 2024-09-13T22:30:53.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:52 smithi118 ceph-mon[25717]: pgmap v281: 97 pgs: 97 active+clean; 234 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 489 op/s 2024-09-13T22:30:53.478 INFO:teuthology.orchestra.run.smithi105.stdout: 5 36329 13.14 MB/sec warmup 81 sec latency 37.601 ms 2024-09-13T22:30:54.435 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:54 smithi105 ceph-mon[23707]: pgmap v282: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.5 MiB/s wr, 606 op/s 2024-09-13T22:30:54.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 36812 13.17 MB/sec warmup 82 sec latency 33.184 ms 2024-09-13T22:30:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:54 smithi118 ceph-mon[25717]: pgmap v282: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.5 MiB/s wr, 606 op/s 2024-09-13T22:30:55.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 37598 13.14 MB/sec warmup 83 sec latency 27.380 ms 2024-09-13T22:30:56.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 38222 13.23 MB/sec warmup 84 sec latency 64.605 ms 2024-09-13T22:30:56.947 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:56 smithi105 ceph-mon[23707]: pgmap v283: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.3 MiB/s wr, 640 op/s 2024-09-13T22:30:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:56 smithi118 ceph-mon[25717]: pgmap v283: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.3 MiB/s wr, 640 op/s 2024-09-13T22:30:57.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 38852 13.40 MB/sec warmup 85 sec latency 32.989 ms 2024-09-13T22:30:58.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 39243 13.40 MB/sec warmup 86 sec latency 54.790 ms 2024-09-13T22:30:58.995 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:30:58 smithi105 ceph-mon[23707]: pgmap v284: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.3 MiB/s wr, 695 op/s 2024-09-13T22:30:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:30:58 smithi118 ceph-mon[25717]: pgmap v284: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.3 MiB/s wr, 695 op/s 2024-09-13T22:30:59.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 39569 13.29 MB/sec warmup 87 sec latency 75.742 ms 2024-09-13T22:31:00.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 39864 13.18 MB/sec warmup 88 sec latency 47.191 ms 2024-09-13T22:31:00.799 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:00 smithi105 ceph-mon[23707]: pgmap v285: 97 pgs: 97 active+clean; 247 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.0 MiB/s wr, 628 op/s 2024-09-13T22:31:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:00 smithi118 ceph-mon[25717]: pgmap v285: 97 pgs: 97 active+clean; 247 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.0 MiB/s wr, 628 op/s 2024-09-13T22:31:01.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 40101 13.08 MB/sec warmup 89 sec latency 45.670 ms 2024-09-13T22:31:02.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 40366 13.05 MB/sec warmup 90 sec latency 177.656 ms 2024-09-13T22:31:02.840 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:31:02.876 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:02 smithi105 ceph-mon[23707]: pgmap v286: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 700 op/s 2024-09-13T22:31:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:02 smithi118 ceph-mon[25717]: pgmap v286: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 700 op/s 2024-09-13T22:31:03.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 40681 12.94 MB/sec warmup 91 sec latency 58.922 ms 2024-09-13T22:31:03.585 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:31:03.585 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (6m) 93s ago 8m 22.2M - ba2b418f427c baade4bebf2e 2024-09-13T22:31:03.585 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (7m) 93s ago 7m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:31:03.585 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (7m) 95s ago 7m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:31:03.585 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (6m) 93s ago 7m 48.9M - 8.3.5 dad864ee21e9 20a1ee977603 2024-09-13T22:31:03.585 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (2m) 93s ago 2m 13.5M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (119s) 95s ago 119s 12.1M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:9283 running (9m) 93s ago 9m 460M - 17.2.0 e1d6a67b021e 3fdf94a6360e 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283 running (7m) 95s ago 7m 411M - 17.2.0 e1d6a67b021e 70a20d7579fb 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (9m) 93s ago 9m 48.1M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (7m) 95s ago 7m 34.4M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (104s) 93s ago 103s 49.0M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (98s) 95s ago 98s 46.7M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (7m) 93s ago 7m 17.7M - 1dbe0e931976 71bdc25e0af6 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (7m) 95s ago 7m 17.2M - 1dbe0e931976 66d8f794097b 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (6m) 93s ago 6m 45.1M 3216M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (6m) 93s ago 6m 45.8M 3216M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (5m) 93s ago 5m 40.1M 3216M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (5m) 93s ago 5m 42.4M 3216M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:31:03.586 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (4m) 95s ago 4m 40.0M 3984M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:31:03.587 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (4m) 95s ago 4m 38.8M 3984M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:31:03.587 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (3m) 95s ago 3m 39.9M 3984M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:31:03.587 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (3m) 95s ago 3m 43.5M 3984M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:31:03.587 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (6m) 93s ago 6m 62.9M - 514e6a882f6e da708a26e310 2024-09-13T22:31:03.901 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:03 smithi105 ceph-mon[23707]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:04.053 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:31:04.053 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:31:04.053 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:31:04.053 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:31:04.053 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:31:04.053 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:31:04.053 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:31:04.054 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:31:04.096 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:03 smithi118 ceph-mon[25717]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:04.426 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:31:04.426 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:31:04.427 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:31:04.427 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [], 2024-09-13T22:31:04.427 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "0/23 daemons upgraded", 2024-09-13T22:31:04.427 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 image on host smithi118" 2024-09-13T22:31:04.427 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:31:04.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 40998 12.89 MB/sec warmup 92 sec latency 76.698 ms 2024-09-13T22:31:04.884 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:31:04.944 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:04 smithi105 ceph-mon[23707]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:04.944 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:04 smithi105 ceph-mon[23707]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:04.944 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:04 smithi105 ceph-mon[23707]: pgmap v287: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 8.0 MiB/s wr, 775 op/s 2024-09-13T22:31:04.944 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:04 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/4037597600' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:31:05.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:04 smithi118 ceph-mon[25717]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:05.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:04 smithi118 ceph-mon[25717]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:05.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:04 smithi118 ceph-mon[25717]: pgmap v287: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 8.0 MiB/s wr, 775 op/s 2024-09-13T22:31:05.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:04 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/4037597600' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:31:05.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 41227 12.78 MB/sec warmup 93 sec latency 51.711 ms 2024-09-13T22:31:06.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:05 smithi105 ceph-mon[23707]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:06.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:05 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1196254336' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:31:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:05 smithi118 ceph-mon[25717]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:05 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/1196254336' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:31:06.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 41556 12.73 MB/sec warmup 94 sec latency 51.957 ms 2024-09-13T22:31:07.041 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:06 smithi105 ceph-mon[23707]: pgmap v288: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 640 op/s 2024-09-13T22:31:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:06 smithi118 ceph-mon[25717]: pgmap v288: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 640 op/s 2024-09-13T22:31:07.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 41959 12.75 MB/sec warmup 95 sec latency 68.259 ms 2024-09-13T22:31:08.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 42508 12.88 MB/sec warmup 96 sec latency 43.053 ms 2024-09-13T22:31:08.820 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:08 smithi105 ceph-mon[23707]: pgmap v289: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.0 MiB/s wr, 556 op/s 2024-09-13T22:31:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:08 smithi118 ceph-mon[25717]: pgmap v289: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.0 MiB/s wr, 556 op/s 2024-09-13T22:31:09.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 42814 12.85 MB/sec warmup 97 sec latency 91.840 ms 2024-09-13T22:31:10.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 43121 12.78 MB/sec warmup 98 sec latency 83.331 ms 2024-09-13T22:31:10.874 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:10 smithi105 ceph-mon[23707]: pgmap v290: 97 pgs: 97 active+clean; 258 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.3 MiB/s wr, 564 op/s 2024-09-13T22:31:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:10 smithi118 ceph-mon[25717]: pgmap v290: 97 pgs: 97 active+clean; 258 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.3 MiB/s wr, 564 op/s 2024-09-13T22:31:11.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 43345 12.67 MB/sec warmup 99 sec latency 91.177 ms 2024-09-13T22:31:12.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 43555 12.59 MB/sec warmup 100 sec latency 92.528 ms 2024-09-13T22:31:13.005 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:12 smithi118 ceph-mon[25717]: pgmap v291: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 542 op/s 2024-09-13T22:31:13.034 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:12 smithi105 ceph-mon[23707]: pgmap v291: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 542 op/s 2024-09-13T22:31:13.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 43972 12.58 MB/sec warmup 101 sec latency 46.188 ms 2024-09-13T22:31:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 44555 12.53 MB/sec warmup 102 sec latency 49.996 ms 2024-09-13T22:31:14.570 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:14 smithi105 ceph-mon[23707]: pgmap v292: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.2 MiB/s wr, 497 op/s 2024-09-13T22:31:14.570 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:14 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:14.570 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:14 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:31:14.570 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:14 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:31:14.570 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:14 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:31:14.756 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:14 smithi118 ceph-mon[25717]: pgmap v292: 97 pgs: 97 active+clean; 270 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.2 MiB/s wr, 497 op/s 2024-09-13T22:31:14.756 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:14 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:14.757 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:14 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:31:14.757 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:14 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:31:14.757 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:14 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:31:15.487 INFO:teuthology.orchestra.run.smithi105.stdout: 5 45023 12.54 MB/sec warmup 103 sec latency 52.521 ms 2024-09-13T22:31:15.599 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:15 smithi105 ceph-mon[23707]: Upgrade: Updating mgr.smithi118.rbhtgz 2024-09-13T22:31:15.599 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:15 smithi105 ceph-mon[23707]: Deploying daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:31:15.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:15 smithi118 ceph-mon[25717]: Upgrade: Updating mgr.smithi118.rbhtgz 2024-09-13T22:31:15.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:15 smithi118 ceph-mon[25717]: Deploying daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:31:16.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 45486 12.57 MB/sec warmup 104 sec latency 74.862 ms 2024-09-13T22:31:16.616 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:16 smithi105 ceph-mon[23707]: pgmap v293: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 492 op/s 2024-09-13T22:31:16.748 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:16 smithi118 ceph-mon[25717]: pgmap v293: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 492 op/s 2024-09-13T22:31:17.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 45856 12.60 MB/sec warmup 105 sec latency 55.879 ms 2024-09-13T22:31:18.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 46283 12.64 MB/sec warmup 106 sec latency 56.474 ms 2024-09-13T22:31:18.905 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:18 smithi105 ceph-mon[23707]: pgmap v294: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.9 MiB/s wr, 600 op/s 2024-09-13T22:31:18.905 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:18.905 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:31:18.905 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:31:18.905 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:18 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:31:19.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:18 smithi118 ceph-mon[25717]: pgmap v294: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.9 MiB/s wr, 600 op/s 2024-09-13T22:31:19.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:19.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:31:19.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:31:19.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:18 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:31:19.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 46732 12.60 MB/sec warmup 107 sec latency 43.224 ms 2024-09-13T22:31:20.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 47198 12.55 MB/sec warmup 108 sec latency 43.094 ms 2024-09-13T22:31:20.951 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:20 smithi105 ceph-mon[23707]: pgmap v295: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.6 MiB/s wr, 509 op/s 2024-09-13T22:31:21.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:20 smithi118 ceph-mon[25717]: pgmap v295: 97 pgs: 97 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.6 MiB/s wr, 509 op/s 2024-09-13T22:31:21.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 47813 12.58 MB/sec warmup 109 sec latency 39.002 ms 2024-09-13T22:31:22.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 48272 12.54 MB/sec warmup 110 sec latency 50.727 ms 2024-09-13T22:31:23.012 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:22 smithi105 ceph-mon[23707]: pgmap v296: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 496 op/s 2024-09-13T22:31:23.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:22 smithi118 ceph-mon[25717]: pgmap v296: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 496 op/s 2024-09-13T22:31:23.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 48818 12.59 MB/sec warmup 111 sec latency 29.404 ms 2024-09-13T22:31:24.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 49427 12.67 MB/sec warmup 112 sec latency 134.044 ms 2024-09-13T22:31:24.524 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:24 smithi105 ceph-mon[23707]: pgmap v297: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.6 MiB/s wr, 611 op/s 2024-09-13T22:31:24.524 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:24 smithi105 ceph-mon[23707]: Standby manager daemon smithi118.rbhtgz restarted 2024-09-13T22:31:24.524 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:24 smithi105 ceph-mon[23707]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:31:24.524 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:24 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:31:24.524 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:24 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:31:24.524 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:24 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:31:24.524 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:24 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:31:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:24 smithi118 ceph-mon[25717]: pgmap v297: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.6 MiB/s wr, 611 op/s 2024-09-13T22:31:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:24 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz restarted 2024-09-13T22:31:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:24 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:31:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:24 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:31:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:24 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:31:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:24 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:31:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:24 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:31:25.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 49813 12.70 MB/sec warmup 113 sec latency 43.927 ms 2024-09-13T22:31:25.537 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:25 smithi105 ceph-mon[23707]: mgrmap e22: smithi105.swmmea(active, since 7m), standbys: smithi118.rbhtgz 2024-09-13T22:31:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:25 smithi118 ceph-mon[25717]: mgrmap e22: smithi105.swmmea(active, since 7m), standbys: smithi118.rbhtgz 2024-09-13T22:31:26.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 50231 12.66 MB/sec warmup 114 sec latency 48.337 ms 2024-09-13T22:31:26.553 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:31:26.554 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:26 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:31:26.554 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:26 smithi105 ceph-mon[23707]: pgmap v298: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 569 op/s 2024-09-13T22:31:26.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:26 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:31:26.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:26 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:31:26.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:26 smithi118 ceph-mon[25717]: pgmap v298: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 569 op/s 2024-09-13T22:31:27.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 50515 12.59 MB/sec warmup 115 sec latency 66.887 ms 2024-09-13T22:31:28.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 50855 12.57 MB/sec warmup 116 sec latency 43.265 ms 2024-09-13T22:31:28.827 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:28 smithi105 ceph-mon[23707]: pgmap v299: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.1 MiB/s wr, 644 op/s 2024-09-13T22:31:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:28 smithi118 ceph-mon[25717]: pgmap v299: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.1 MiB/s wr, 644 op/s 2024-09-13T22:31:29.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 51359 12.53 MB/sec warmup 117 sec latency 37.512 ms 2024-09-13T22:31:30.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 51898 12.52 MB/sec warmup 118 sec latency 65.013 ms 2024-09-13T22:31:30.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:30 smithi118 ceph-mon[25717]: pgmap v300: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.2 MiB/s wr, 553 op/s 2024-09-13T22:31:30.874 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:30 smithi105 ceph-mon[23707]: pgmap v300: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.2 MiB/s wr, 553 op/s 2024-09-13T22:31:31.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 52331 12.55 MB/sec warmup 119 sec latency 96.885 ms 2024-09-13T22:31:32.923 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:32 smithi105 ceph-mon[23707]: pgmap v301: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 616 op/s 2024-09-13T22:31:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:32 smithi118 ceph-mon[25717]: pgmap v301: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 616 op/s 2024-09-13T22:31:33.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 53176 12.11 MB/sec execute 1 sec latency 56.761 ms 2024-09-13T22:31:34.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 53371 9.23 MB/sec execute 2 sec latency 79.646 ms 2024-09-13T22:31:34.495 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:34 smithi105 ceph-mon[23707]: pgmap v302: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.7 MiB/s wr, 776 op/s 2024-09-13T22:31:34.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:34 smithi118 ceph-mon[25717]: pgmap v302: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.7 MiB/s wr, 776 op/s 2024-09-13T22:31:35.260 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:31:35.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 53593 7.23 MB/sec execute 3 sec latency 72.659 ms 2024-09-13T22:31:35.978 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (7m) 2m ago 8m 22.2M - ba2b418f427c baade4bebf2e 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (8m) 2m ago 8m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (8m) 2m ago 8m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (6m) 2m ago 8m 48.9M - 8.3.5 dad864ee21e9 20a1ee977603 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (2m) 2m ago 2m 13.5M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (2m) 2m ago 2m 12.1M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e 3fdf94a6360e 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283 starting - - - - 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (9m) 2m ago 10m 48.1M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (7m) 2m ago 7m 34.4M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (2m) 2m ago 2m 49.0M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:31:35.979 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (2m) 2m ago 2m 46.7M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (8m) 2m ago 8m 17.7M - 1dbe0e931976 71bdc25e0af6 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (7m) 2m ago 7m 17.2M - 1dbe0e931976 66d8f794097b 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (7m) 2m ago 7m 45.1M 3216M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (6m) 2m ago 6m 45.8M 3216M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (6m) 2m ago 6m 40.1M 3216M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (5m) 2m ago 5m 42.4M 3216M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (5m) 2m ago 5m 40.0M 3984M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (4m) 2m ago 4m 38.8M 3984M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (4m) 2m ago 4m 39.9M 3984M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (4m) 2m ago 4m 43.5M 3984M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:31:35.980 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (7m) 2m ago 7m 62.9M - 514e6a882f6e da708a26e310 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:31:36.417 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:31:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:31:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:31:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:31:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:31:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:31:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:31:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:31:36.418 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:31:36.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 53842 6.67 MB/sec execute 4 sec latency 115.770 ms 2024-09-13T22:31:36.779 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:31:36.779 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:31:36.779 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:31:36.779 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [], 2024-09-13T22:31:36.779 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "", 2024-09-13T22:31:36.779 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading mgr daemons" 2024-09-13T22:31:36.779 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:31:36.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:36 smithi105 ceph-mon[23707]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:36.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:36 smithi105 ceph-mon[23707]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:36.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:36 smithi105 ceph-mon[23707]: pgmap v303: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 645 op/s 2024-09-13T22:31:36.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:36 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2305015593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:31:37.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:36 smithi118 ceph-mon[25717]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:37.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:36 smithi118 ceph-mon[25717]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:37.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:36 smithi118 ceph-mon[25717]: pgmap v303: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 645 op/s 2024-09-13T22:31:37.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:36 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2305015593' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:31:37.268 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:31:37.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 54142 6.54 MB/sec execute 5 sec latency 53.880 ms 2024-09-13T22:31:37.840 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:37 smithi105 ceph-mon[23707]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:37.840 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:37 smithi105 ceph-mon[23707]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:37.840 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:37 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/115108646' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:31:37.840 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:37 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:38.029 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:37 smithi118 ceph-mon[25717]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:38.029 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:37 smithi118 ceph-mon[25717]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:31:38.029 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:37 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/115108646' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:31:38.029 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:37 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:38.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 54512 7.33 MB/sec execute 6 sec latency 56.045 ms 2024-09-13T22:31:38.847 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:38 smithi105 ceph-mon[23707]: pgmap v304: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.0 MiB/s wr, 660 op/s 2024-09-13T22:31:38.848 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:38 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:38.944 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:38 smithi118 ceph-mon[25717]: pgmap v304: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.0 MiB/s wr, 660 op/s 2024-09-13T22:31:38.944 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:38 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:39.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 55076 7.25 MB/sec execute 7 sec latency 53.044 ms 2024-09-13T22:31:40.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 55520 7.73 MB/sec execute 8 sec latency 50.609 ms 2024-09-13T22:31:40.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:40 smithi118 ceph-mon[25717]: pgmap v305: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 499 op/s 2024-09-13T22:31:41.008 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:40 smithi105 ceph-mon[23707]: pgmap v305: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 499 op/s 2024-09-13T22:31:41.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 56035 8.75 MB/sec execute 9 sec latency 53.913 ms 2024-09-13T22:31:42.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 56562 10.07 MB/sec execute 10 sec latency 43.495 ms 2024-09-13T22:31:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:42 smithi105 ceph-mon[23707]: pgmap v306: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 611 op/s 2024-09-13T22:31:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:31:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:42 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr fail", "who": "smithi105.swmmea"}]: dispatch 2024-09-13T22:31:42.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:42 smithi105 ceph-mon[23707]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T22:31:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:42 smithi118 ceph-mon[25717]: pgmap v306: 97 pgs: 97 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 611 op/s 2024-09-13T22:31:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' 2024-09-13T22:31:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:31:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:42 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr fail", "who": "smithi105.swmmea"}]: dispatch 2024-09-13T22:31:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:42 smithi118 ceph-mon[25717]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T22:31:43.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 56933 10.45 MB/sec execute 11 sec latency 51.894 ms 2024-09-13T22:31:43.817 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: Upgrade: Need to upgrade myself (mgr.smithi105.swmmea) 2024-09-13T22:31:43.817 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: Failing over to other MGR 2024-09-13T22:31:43.817 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: Standby manager daemon smithi105.swmmea started 2024-09-13T22:31:43.817 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:31:43.817 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "mgr fail", "who": "smithi105.swmmea"}]': finished 2024-09-13T22:31:43.817 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: mgrmap e23: smithi118.rbhtgz(active, starting, since 1.00243s), standbys: smithi105.swmmea 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rbhtgz", "id": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:31:43.818 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: Manager daemon smithi118.rbhtgz is now available 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: Checking for cert/key for grafana.smithi105 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rbhtgz/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rbhtgz/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rbhtgz/trash_purge_schedule"}]: dispatch 2024-09-13T22:31:43.819 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:43 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rbhtgz/trash_purge_schedule"}]: dispatch 2024-09-13T22:31:43.842 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: Upgrade: Need to upgrade myself (mgr.smithi105.swmmea) 2024-09-13T22:31:43.842 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: Failing over to other MGR 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: Standby manager daemon smithi105.swmmea started 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.14202 172.21.15.105:0/3270011726' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "mgr fail", "who": "smithi105.swmmea"}]': finished 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: mgrmap e23: smithi118.rbhtgz(active, starting, since 1.00243s), standbys: smithi105.swmmea 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rbhtgz", "id": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:31:43.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: Manager daemon smithi118.rbhtgz is now available 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: Checking for cert/key for grafana.smithi105 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:31:43.844 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:31:43.845 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rbhtgz/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:31:43.845 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rbhtgz/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:31:43.845 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rbhtgz/trash_purge_schedule"}]: dispatch 2024-09-13T22:31:43.845 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:43 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rbhtgz/trash_purge_schedule"}]: dispatch 2024-09-13T22:31:44.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 57173 9.78 MB/sec execute 12 sec latency 63.120 ms 2024-09-13T22:31:45.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 57506 9.55 MB/sec execute 13 sec latency 78.091 ms 2024-09-13T22:31:45.626 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:45 smithi105 ceph-mon[23707]: Deploying cephadm binary to smithi118 2024-09-13T22:31:45.626 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:45 smithi105 ceph-mon[23707]: Deploying cephadm binary to smithi105 2024-09-13T22:31:45.626 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:45 smithi105 ceph-mon[23707]: mgrmap e24: smithi118.rbhtgz(active, since 2s), standbys: smithi105.swmmea 2024-09-13T22:31:45.626 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:45 smithi105 ceph-mon[23707]: pgmap v3: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T22:31:45.626 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:45 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:45.626 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:45 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:45.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:45 smithi118 ceph-mon[25717]: Deploying cephadm binary to smithi118 2024-09-13T22:31:45.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:45 smithi118 ceph-mon[25717]: Deploying cephadm binary to smithi105 2024-09-13T22:31:45.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:45 smithi118 ceph-mon[25717]: mgrmap e24: smithi118.rbhtgz(active, since 2s), standbys: smithi105.swmmea 2024-09-13T22:31:45.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:45 smithi118 ceph-mon[25717]: pgmap v3: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T22:31:45.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:45 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:45.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:45 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:46.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 57719 9.20 MB/sec execute 14 sec latency 56.587 ms 2024-09-13T22:31:46.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:46 smithi105 ceph-mon[23707]: pgmap v4: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T22:31:46.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:46 smithi105 ceph-mon[23707]: [13/Sep/2024:22:31:45] ENGINE Bus STARTING 2024-09-13T22:31:46.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:46 smithi105 ceph-mon[23707]: [13/Sep/2024:22:31:45] ENGINE Serving on https://172.21.15.118:7150 2024-09-13T22:31:46.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:46 smithi105 ceph-mon[23707]: [13/Sep/2024:22:31:45] ENGINE Error in HTTPServer.serve 2024-09-13T22:31:46.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Traceback (most recent call last): 2024-09-13T22:31:46.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:31:46.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:31:46.642 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self._run(expiration_interval) 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: s = self.context.wrap_socket( 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: return self.sslsocket_class._create( 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self.do_handshake() 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:31:46.643 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self._sslobj.do_handshake() 2024-09-13T22:31:46.644 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:31:46.644 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:46 smithi105 ceph-mon[23707]: [13/Sep/2024:22:31:45] ENGINE Serving on http://172.21.15.118:8765 2024-09-13T22:31:46.644 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:46 smithi105 ceph-mon[23707]: [13/Sep/2024:22:31:45] ENGINE Bus STARTED 2024-09-13T22:31:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:46 smithi118 ceph-mon[25717]: pgmap v4: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T22:31:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:46 smithi118 ceph-mon[25717]: [13/Sep/2024:22:31:45] ENGINE Bus STARTING 2024-09-13T22:31:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:46 smithi118 ceph-mon[25717]: [13/Sep/2024:22:31:45] ENGINE Serving on https://172.21.15.118:7150 2024-09-13T22:31:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:46 smithi118 ceph-mon[25717]: [13/Sep/2024:22:31:45] ENGINE Error in HTTPServer.serve 2024-09-13T22:31:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Traceback (most recent call last): 2024-09-13T22:31:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:31:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._run(expiration_interval) 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s = self.context.wrap_socket( 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: return self.sslsocket_class._create( 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self.do_handshake() 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:31:46.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._sslobj.do_handshake() 2024-09-13T22:31:46.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:31:46.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:46 smithi118 ceph-mon[25717]: [13/Sep/2024:22:31:45] ENGINE Serving on http://172.21.15.118:8765 2024-09-13T22:31:46.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:46 smithi118 ceph-mon[25717]: [13/Sep/2024:22:31:45] ENGINE Bus STARTED 2024-09-13T22:31:47.488 INFO:teuthology.orchestra.run.smithi105.stdout: 5 58092 9.31 MB/sec execute 15 sec latency 56.513 ms 2024-09-13T22:31:47.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:47 smithi118 ceph-mon[25717]: mgrmap e25: smithi118.rbhtgz(active, since 4s), standbys: smithi105.swmmea 2024-09-13T22:31:47.672 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:47 smithi105 ceph-mon[23707]: mgrmap e25: smithi118.rbhtgz(active, since 4s), standbys: smithi105.swmmea 2024-09-13T22:31:48.489 INFO:teuthology.orchestra.run.smithi105.stdout: 5 58588 9.08 MB/sec execute 16 sec latency 54.110 ms 2024-09-13T22:31:48.704 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:48 smithi105 ceph-mon[23707]: pgmap v5: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T22:31:48.835 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:48 smithi118 ceph-mon[25717]: pgmap v5: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T22:31:49.489 INFO:teuthology.orchestra.run.smithi105.stdout: 5 58924 8.89 MB/sec execute 17 sec latency 76.754 ms 2024-09-13T22:31:50.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:50 smithi105 ceph-mon[23707]: pgmap v6: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T22:31:50.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:50 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:50.482 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:50 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:50.491 INFO:teuthology.orchestra.run.smithi105.stdout: 5 59276 9.22 MB/sec execute 18 sec latency 55.741 ms 2024-09-13T22:31:50.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:50 smithi118 ceph-mon[25717]: pgmap v6: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T22:31:50.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:50 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:50.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:50 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:51.491 INFO:teuthology.orchestra.run.smithi105.stdout: 5 59815 9.68 MB/sec execute 19 sec latency 52.757 ms 2024-09-13T22:31:52.010 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:51 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:52.010 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:51 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:52.010 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:51 smithi105 ceph-mon[23707]: pgmap v7: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 3.1 MiB/s wr, 407 op/s 2024-09-13T22:31:52.010 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:51 smithi105 ceph-mon[23707]: Standby manager daemon smithi105.swmmea restarted 2024-09-13T22:31:52.010 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:51 smithi105 ceph-mon[23707]: Standby manager daemon smithi105.swmmea started 2024-09-13T22:31:52.010 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:51 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.105:0/3800458290' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi105.swmmea/crt"}]: dispatch 2024-09-13T22:31:52.011 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:51 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.105:0/3800458290' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:31:52.011 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:51 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.105:0/3800458290' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi105.swmmea/key"}]: dispatch 2024-09-13T22:31:52.011 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:51 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.105:0/3800458290' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:31:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:51 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:51 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:51 smithi118 ceph-mon[25717]: pgmap v7: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 3.1 MiB/s wr, 407 op/s 2024-09-13T22:31:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:51 smithi118 ceph-mon[25717]: Standby manager daemon smithi105.swmmea restarted 2024-09-13T22:31:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:51 smithi118 ceph-mon[25717]: Standby manager daemon smithi105.swmmea started 2024-09-13T22:31:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:51 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.105:0/3800458290' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi105.swmmea/crt"}]: dispatch 2024-09-13T22:31:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:51 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.105:0/3800458290' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:31:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:51 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.105:0/3800458290' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi105.swmmea/key"}]: dispatch 2024-09-13T22:31:52.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:51 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.105:0/3800458290' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:31:52.491 INFO:teuthology.orchestra.run.smithi105.stdout: 5 60288 10.16 MB/sec execute 20 sec latency 97.396 ms 2024-09-13T22:31:52.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:52 smithi118 ceph-mon[25717]: mgrmap e26: smithi118.rbhtgz(active, since 9s), standbys: smithi105.swmmea 2024-09-13T22:31:53.020 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:52 smithi105 ceph-mon[23707]: mgrmap e26: smithi118.rbhtgz(active, since 9s), standbys: smithi105.swmmea 2024-09-13T22:31:53.492 INFO:teuthology.orchestra.run.smithi105.stdout: 5 60671 10.12 MB/sec execute 21 sec latency 43.262 ms 2024-09-13T22:31:54.502 INFO:teuthology.orchestra.run.smithi105.stdout: 5 61030 9.97 MB/sec execute 22 sec latency 81.644 ms 2024-09-13T22:31:54.547 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: pgmap v8: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.4 MiB/s wr, 313 op/s 2024-09-13T22:31:54.547 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.547 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.547 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.547 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:54 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: pgmap v8: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.4 MiB/s wr, 313 op/s 2024-09-13T22:31:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:54 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:55.502 INFO:teuthology.orchestra.run.smithi105.stdout: 5 61306 9.85 MB/sec execute 23 sec latency 63.765 ms 2024-09-13T22:31:55.559 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: Adjusting osd_memory_target on smithi105 to 1936M 2024-09-13T22:31:55.559 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:55.559 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.559 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: Adjusting osd_memory_target on smithi118 to 2704M 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-13T22:31:55.560 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:31:55.561 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:31:55.561 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:55.561 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:55.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: Adjusting osd_memory_target on smithi105 to 1936M 2024-09-13T22:31:55.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:55.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: Adjusting osd_memory_target on smithi118 to 2704M 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:31:55.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:31:55.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:55.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:56.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 61742 9.81 MB/sec execute 24 sec latency 31.870 ms 2024-09-13T22:31:56.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:31:56.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:31:56.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.3 MiB/s wr, 528 op/s 2024-09-13T22:31:56.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:31:56.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:31:56.824 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:56.825 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:56.825 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:56.825 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:56.825 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:56.825 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:56.825 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:56.825 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:56 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:31:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:31:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.3 MiB/s wr, 528 op/s 2024-09-13T22:31:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:31:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:31:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:57.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:57.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:57.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:57.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:57.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:57.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:57.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:56 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:31:57.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 62204 9.68 MB/sec execute 25 sec latency 36.408 ms 2024-09-13T22:31:57.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:57 smithi105 ceph-mon[23707]: Reconfiguring alertmanager.smithi105 (dependencies changed)... 2024-09-13T22:31:57.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:57 smithi105 ceph-mon[23707]: Deploying daemon alertmanager.smithi105 on smithi105 2024-09-13T22:31:58.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:57 smithi118 ceph-mon[25717]: Reconfiguring alertmanager.smithi105 (dependencies changed)... 2024-09-13T22:31:58.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:57 smithi118 ceph-mon[25717]: Deploying daemon alertmanager.smithi105 on smithi105 2024-09-13T22:31:58.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 62575 9.69 MB/sec execute 26 sec latency 55.899 ms 2024-09-13T22:31:58.873 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:58 smithi105 ceph-mon[23707]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.7 MiB/s wr, 471 op/s 2024-09-13T22:31:58.873 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:31:58 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:31:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:58 smithi118 ceph-mon[25717]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.7 MiB/s wr, 471 op/s 2024-09-13T22:31:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:31:58 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:31:59.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 62990 9.96 MB/sec execute 27 sec latency 46.276 ms 2024-09-13T22:32:00.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 63528 10.25 MB/sec execute 28 sec latency 34.775 ms 2024-09-13T22:32:00.928 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:00 smithi105 ceph-mon[23707]: pgmap v11: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.7 MiB/s wr, 471 op/s 2024-09-13T22:32:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:00 smithi118 ceph-mon[25717]: pgmap v11: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.7 MiB/s wr, 471 op/s 2024-09-13T22:32:01.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 63864 10.31 MB/sec execute 29 sec latency 54.260 ms 2024-09-13T22:32:02.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 64102 10.18 MB/sec execute 30 sec latency 89.142 ms 2024-09-13T22:32:02.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:02 smithi105 ceph-mon[23707]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 6.9 MiB/s wr, 709 op/s 2024-09-13T22:32:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:02 smithi118 ceph-mon[25717]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 6.9 MiB/s wr, 709 op/s 2024-09-13T22:32:03.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 64350 9.96 MB/sec execute 31 sec latency 49.320 ms 2024-09-13T22:32:04.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 64647 9.86 MB/sec execute 32 sec latency 48.700 ms 2024-09-13T22:32:04.846 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:04 smithi105 ceph-mon[23707]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 481 op/s 2024-09-13T22:32:05.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:04 smithi118 ceph-mon[25717]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 481 op/s 2024-09-13T22:32:05.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 64960 9.90 MB/sec execute 33 sec latency 43.769 ms 2024-09-13T22:32:06.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 65410 9.85 MB/sec execute 34 sec latency 45.190 ms 2024-09-13T22:32:06.908 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:06 smithi105 ceph-mon[23707]: pgmap v14: 97 pgs: 97 active+clean; 346 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.4 MiB/s wr, 736 op/s 2024-09-13T22:32:07.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:06 smithi118 ceph-mon[25717]: pgmap v14: 97 pgs: 97 active+clean; 346 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.4 MiB/s wr, 736 op/s 2024-09-13T22:32:07.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 65851 9.70 MB/sec execute 35 sec latency 82.735 ms 2024-09-13T22:32:07.842 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:32:08.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 66199 9.72 MB/sec execute 36 sec latency 62.232 ms 2024-09-13T22:32:08.817 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:32:08.817 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (7m) 15s ago 9m 26.1M - 0.23.0 ba2b418f427c baade4bebf2e 2024-09-13T22:32:08.817 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (9m) 15s ago 9m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:32:08.817 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (8m) 18s ago 8m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:32:08.817 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (7m) 15s ago 8m 49.2M - 8.3.5 dad864ee21e9 20a1ee977603 2024-09-13T22:32:08.818 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (3m) 15s ago 3m 34.9M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:32:08.818 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (3m) 18s ago 3m 13.0M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:32:08.818 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:9283 running (10m) 15s ago 10m 167M - 17.2.0 e1d6a67b021e 3fdf94a6360e 2024-09-13T22:32:08.818 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283 running (51s) 18s ago 8m 484M - 19.1.1-324-gecab7be5 0652ff2b5d78 25b0284f8ebd 2024-09-13T22:32:08.818 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (10m) 15s ago 10m 50.4M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:32:08.818 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (8m) 18s ago 8m 39.1M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:32:08.818 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (2m) 15s ago 2m 239M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:32:08.819 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (2m) 18s ago 2m 74.5M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:32:08.819 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (8m) 15s ago 8m 19.2M - 1.3.1 1dbe0e931976 71bdc25e0af6 2024-09-13T22:32:08.819 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (8m) 18s ago 8m 17.2M - 1.3.1 1dbe0e931976 66d8f794097b 2024-09-13T22:32:08.819 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (7m) 15s ago 7m 282M 1936M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:32:08.819 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (7m) 15s ago 7m 205M 1936M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:32:08.819 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (6m) 15s ago 6m 173M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:32:08.820 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (6m) 15s ago 6m 214M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:32:08.820 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (5m) 18s ago 5m 236M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:32:08.820 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (5m) 18s ago 5m 249M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:32:08.820 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (5m) 18s ago 5m 240M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:32:08.820 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (4m) 18s ago 4m 326M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:32:08.820 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (8m) 15s ago 8m 67.3M - 2.33.4 514e6a882f6e da708a26e310 2024-09-13T22:32:08.920 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:08 smithi105 ceph-mon[23707]: pgmap v15: 97 pgs: 97 active+clean; 346 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 492 op/s 2024-09-13T22:32:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:08 smithi118 ceph-mon[25717]: pgmap v15: 97 pgs: 97 active+clean; 346 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 492 op/s 2024-09-13T22:32:09.395 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:32:09.395 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:32:09.395 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:32:09.395 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:32:09.395 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:32:09.395 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T22:32:09.395 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:32:09.395 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:32:09.395 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:32:09.396 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:32:09.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 66679 10.06 MB/sec execute 37 sec latency 79.562 ms 2024-09-13T22:32:09.950 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:32:09.950 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:32:09.950 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:32:09.950 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:32:09.950 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [], 2024-09-13T22:32:09.950 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "1/23 daemons upgraded", 2024-09-13T22:32:09.950 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "", 2024-09-13T22:32:09.951 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:32:09.951 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:32:09.955 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:09 smithi105 ceph-mon[23707]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:09.955 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:09 smithi105 ceph-mon[23707]: from='client.24473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:09.955 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:09 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2830908687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:32:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:09 smithi118 ceph-mon[25717]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:09 smithi118 ceph-mon[25717]: from='client.24473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:09 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2830908687' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:32:10.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 67152 10.11 MB/sec execute 38 sec latency 32.054 ms 2024-09-13T22:32:10.576 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:32:10.959 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:10 smithi105 ceph-mon[23707]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:10.959 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:10 smithi105 ceph-mon[23707]: pgmap v16: 97 pgs: 97 active+clean; 346 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 492 op/s 2024-09-13T22:32:10.959 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:10 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3957938300' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:32:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:10 smithi118 ceph-mon[25717]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:10 smithi118 ceph-mon[25717]: pgmap v16: 97 pgs: 97 active+clean; 346 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 492 op/s 2024-09-13T22:32:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:10 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3957938300' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:32:11.512 INFO:teuthology.orchestra.run.smithi105.stdout: 5 67526 10.17 MB/sec execute 39 sec latency 32.886 ms 2024-09-13T22:32:11.969 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:11 smithi105 ceph-mon[23707]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:11 smithi118 ceph-mon[25717]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:12.513 INFO:teuthology.orchestra.run.smithi105.stdout: 5 67935 10.11 MB/sec execute 40 sec latency 42.117 ms 2024-09-13T22:32:12.987 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:12 smithi105 ceph-mon[23707]: pgmap v17: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 747 op/s 2024-09-13T22:32:12.987 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:12 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:12.987 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:12 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:12.987 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:12 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:12.987 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:12 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:12.987 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:12 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:32:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:12 smithi118 ceph-mon[25717]: pgmap v17: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 747 op/s 2024-09-13T22:32:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:12 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:12 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:12 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:12 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:12 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:32:13.513 INFO:teuthology.orchestra.run.smithi105.stdout: 5 68397 10.16 MB/sec execute 41 sec latency 53.450 ms 2024-09-13T22:32:14.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:13 smithi105 ceph-mon[23707]: Reconfiguring grafana.smithi105 (dependencies changed)... 2024-09-13T22:32:14.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:13 smithi105 ceph-mon[23707]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T22:32:14.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:13 smithi105 ceph-mon[23707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:32:14.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:13 smithi105 ceph-mon[23707]: Reconfiguring daemon grafana.smithi105 on smithi105 2024-09-13T22:32:14.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:13 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:32:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:13 smithi118 ceph-mon[25717]: Reconfiguring grafana.smithi105 (dependencies changed)... 2024-09-13T22:32:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:13 smithi118 ceph-mon[25717]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T22:32:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:13 smithi118 ceph-mon[25717]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T22:32:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:13 smithi118 ceph-mon[25717]: Reconfiguring daemon grafana.smithi105 on smithi105 2024-09-13T22:32:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:13 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:32:14.513 INFO:teuthology.orchestra.run.smithi105.stdout: 5 68919 10.25 MB/sec execute 42 sec latency 41.331 ms 2024-09-13T22:32:15.013 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:14 smithi105 ceph-mon[23707]: pgmap v18: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 509 op/s 2024-09-13T22:32:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:14 smithi118 ceph-mon[25717]: pgmap v18: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 509 op/s 2024-09-13T22:32:15.513 INFO:teuthology.orchestra.run.smithi105.stdout: 5 69360 10.11 MB/sec execute 43 sec latency 54.503 ms 2024-09-13T22:32:15.789 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:15 smithi105 ceph-mon[23707]: pgmap v19: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 779 op/s 2024-09-13T22:32:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:15 smithi118 ceph-mon[25717]: pgmap v19: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 779 op/s 2024-09-13T22:32:16.513 INFO:teuthology.orchestra.run.smithi105.stdout: 5 69928 10.29 MB/sec execute 44 sec latency 62.211 ms 2024-09-13T22:32:17.513 INFO:teuthology.orchestra.run.smithi105.stdout: 5 70649 10.67 MB/sec execute 45 sec latency 66.918 ms 2024-09-13T22:32:18.360 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:18 smithi105 ceph-mon[23707]: pgmap v20: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 525 op/s 2024-09-13T22:32:18.513 INFO:teuthology.orchestra.run.smithi105.stdout: 5 70984 10.67 MB/sec execute 46 sec latency 115.130 ms 2024-09-13T22:32:18.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:18 smithi118 ceph-mon[25717]: pgmap v20: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 525 op/s 2024-09-13T22:32:19.522 INFO:teuthology.orchestra.run.smithi105.stdout: 5 71319 10.59 MB/sec execute 47 sec latency 44.026 ms 2024-09-13T22:32:20.396 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:20 smithi105 ceph-mon[23707]: pgmap v21: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 525 op/s 2024-09-13T22:32:20.396 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:20 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:20.396 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:20 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:20.396 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:20 smithi105 ceph-mon[23707]: Reconfiguring node-exporter.smithi105 (dependencies changed)... 2024-09-13T22:32:20.396 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:20 smithi105 ceph-mon[23707]: Deploying daemon node-exporter.smithi105 on smithi105 2024-09-13T22:32:20.522 INFO:teuthology.orchestra.run.smithi105.stdout: 5 71658 10.50 MB/sec execute 48 sec latency 37.128 ms 2024-09-13T22:32:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:20 smithi118 ceph-mon[25717]: pgmap v21: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 525 op/s 2024-09-13T22:32:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:20 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:20 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:20 smithi118 ceph-mon[25717]: Reconfiguring node-exporter.smithi105 (dependencies changed)... 2024-09-13T22:32:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:20 smithi118 ceph-mon[25717]: Deploying daemon node-exporter.smithi105 on smithi105 2024-09-13T22:32:21.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 71929 10.45 MB/sec execute 49 sec latency 91.158 ms 2024-09-13T22:32:22.492 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:22 smithi105 ceph-mon[23707]: pgmap v22: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 814 op/s 2024-09-13T22:32:22.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 72226 10.37 MB/sec execute 50 sec latency 59.981 ms 2024-09-13T22:32:22.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:22 smithi118 ceph-mon[25717]: pgmap v22: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 814 op/s 2024-09-13T22:32:23.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 72664 10.36 MB/sec execute 51 sec latency 49.174 ms 2024-09-13T22:32:24.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 73185 10.34 MB/sec execute 52 sec latency 43.654 ms 2024-09-13T22:32:24.547 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:24 smithi105 ceph-mon[23707]: pgmap v23: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 559 op/s 2024-09-13T22:32:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:24 smithi118 ceph-mon[25717]: pgmap v23: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 559 op/s 2024-09-13T22:32:25.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 73594 10.43 MB/sec execute 53 sec latency 32.740 ms 2024-09-13T22:32:26.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 74082 10.61 MB/sec execute 54 sec latency 37.093 ms 2024-09-13T22:32:26.584 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:26 smithi105 ceph-mon[23707]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 782 op/s 2024-09-13T22:32:26.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:26 smithi118 ceph-mon[25717]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 782 op/s 2024-09-13T22:32:27.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 74431 10.57 MB/sec execute 55 sec latency 43.485 ms 2024-09-13T22:32:27.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:27 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:27.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:27 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:27.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:27 smithi105 ceph-mon[23707]: Reconfiguring prometheus.smithi105 (dependencies changed)... 2024-09-13T22:32:28.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:27 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:28.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:27 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:28.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:27 smithi118 ceph-mon[25717]: Reconfiguring prometheus.smithi105 (dependencies changed)... 2024-09-13T22:32:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 74715 10.51 MB/sec execute 56 sec latency 43.419 ms 2024-09-13T22:32:28.875 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:28 smithi105 ceph-mon[23707]: Deploying daemon prometheus.smithi105 on smithi105 2024-09-13T22:32:28.875 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:28 smithi105 ceph-mon[23707]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-09-13T22:32:28.876 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:28 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:32:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:28 smithi118 ceph-mon[25717]: Deploying daemon prometheus.smithi105 on smithi105 2024-09-13T22:32:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:28 smithi118 ceph-mon[25717]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-09-13T22:32:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:28 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:32:29.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 74960 10.43 MB/sec execute 57 sec latency 39.638 ms 2024-09-13T22:32:30.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 75342 10.36 MB/sec execute 58 sec latency 54.768 ms 2024-09-13T22:32:30.933 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:30 smithi105 ceph-mon[23707]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-09-13T22:32:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:30 smithi118 ceph-mon[25717]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-09-13T22:32:31.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 75773 10.40 MB/sec execute 59 sec latency 71.990 ms 2024-09-13T22:32:32.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 76167 10.40 MB/sec execute 60 sec latency 68.315 ms 2024-09-13T22:32:32.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:32 smithi105 ceph-mon[23707]: pgmap v27: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 726 op/s 2024-09-13T22:32:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:32 smithi118 ceph-mon[25717]: pgmap v27: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 726 op/s 2024-09-13T22:32:33.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 76465 10.27 MB/sec execute 61 sec latency 52.391 ms 2024-09-13T22:32:34.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 76832 10.30 MB/sec execute 62 sec latency 60.092 ms 2024-09-13T22:32:34.817 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:34 smithi105 ceph-mon[23707]: pgmap v28: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 437 op/s 2024-09-13T22:32:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:34 smithi118 ceph-mon[25717]: pgmap v28: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 437 op/s 2024-09-13T22:32:35.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 77225 10.34 MB/sec execute 63 sec latency 43.627 ms 2024-09-13T22:32:36.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 77607 10.43 MB/sec execute 64 sec latency 37.739 ms 2024-09-13T22:32:36.918 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:36 smithi105 ceph-mon[23707]: pgmap v29: 97 pgs: 97 active+clean; 339 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 664 op/s 2024-09-13T22:32:37.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:36 smithi118 ceph-mon[25717]: pgmap v29: 97 pgs: 97 active+clean; 339 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 664 op/s 2024-09-13T22:32:37.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 77936 10.40 MB/sec execute 65 sec latency 44.055 ms 2024-09-13T22:32:38.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 78217 10.34 MB/sec execute 66 sec latency 84.087 ms 2024-09-13T22:32:38.994 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:38 smithi105 ceph-mon[23707]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-13T22:32:39.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:38 smithi118 ceph-mon[25717]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-13T22:32:39.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 78542 10.29 MB/sec execute 67 sec latency 87.543 ms 2024-09-13T22:32:40.012 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:39 smithi105 ceph-mon[23707]: pgmap v31: 97 pgs: 97 active+clean; 339 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-13T22:32:40.096 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:39 smithi118 ceph-mon[25717]: pgmap v31: 97 pgs: 97 active+clean; 339 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 441 op/s 2024-09-13T22:32:40.523 INFO:teuthology.orchestra.run.smithi105.stdout: 5 78984 10.34 MB/sec execute 68 sec latency 54.945 ms 2024-09-13T22:32:41.130 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:32:41.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 79507 10.34 MB/sec execute 69 sec latency 43.324 ms 2024-09-13T22:32:42.275 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:42 smithi105 ceph-mon[23707]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:42.275 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:42 smithi105 ceph-mon[23707]: pgmap v32: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.9 MiB/s wr, 640 op/s 2024-09-13T22:32:42.275 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:42 smithi105 ceph-mon[23707]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 starting - - - - 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (9m) 48s ago 9m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (9m) 52s ago 9m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 starting - - - - 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (3m) 48s ago 3m 34.9M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (3m) 52s ago 3m 13.0M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:9283 running (10m) 48s ago 10m 167M - 17.2.0 e1d6a67b021e 3fdf94a6360e 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283 running (84s) 52s ago 9m 484M - 19.1.1-324-gecab7be5 0652ff2b5d78 25b0284f8ebd 2024-09-13T22:32:42.295 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (10m) 48s ago 11m 50.4M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (8m) 52s ago 8m 39.1M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (3m) 48s ago 3m 239M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (3m) 52s ago 3m 74.5M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 starting - - - - 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (8m) 52s ago 8m 17.2M - 1.3.1 1dbe0e931976 66d8f794097b 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (8m) 48s ago 8m 282M 1936M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (7m) 48s ago 7m 205M 1936M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (7m) 48s ago 7m 173M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (6m) 48s ago 6m 214M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (6m) 52s ago 6m 236M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:32:42.296 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (6m) 52s ago 6m 249M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:32:42.297 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (5m) 52s ago 5m 240M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:32:42.297 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (5m) 52s ago 5m 326M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:32:42.297 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (8m) 48s ago 8m 67.3M - 2.33.4 514e6a882f6e da708a26e310 2024-09-13T22:32:42.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 79984 10.31 MB/sec execute 70 sec latency 72.686 ms 2024-09-13T22:32:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:42 smithi118 ceph-mon[25717]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:42 smithi118 ceph-mon[25717]: pgmap v32: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.9 MiB/s wr, 640 op/s 2024-09-13T22:32:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:42 smithi118 ceph-mon[25717]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:43.006 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:32:43.006 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:32:43.006 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:32:43.006 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:32:43.006 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:32:43.007 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T22:32:43.008 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:32:43.008 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:32:43.008 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:32:43.286 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:43 smithi105 ceph-mon[23707]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:43.286 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:43 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/3554658031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:32:43.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 80467 10.37 MB/sec execute 71 sec latency 64.199 ms 2024-09-13T22:32:43.587 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:32:43.587 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:32:43.587 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:32:43.587 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:32:43.588 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [], 2024-09-13T22:32:43.588 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "1/23 daemons upgraded", 2024-09-13T22:32:43.588 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "", 2024-09-13T22:32:43.588 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:32:43.588 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:32:43.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:43 smithi118 ceph-mon[25717]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:43.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:43 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3554658031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:32:44.232 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:32:44.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 80955 10.49 MB/sec execute 72 sec latency 43.221 ms 2024-09-13T22:32:44.570 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:44 smithi105 ceph-mon[23707]: pgmap v33: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-09-13T22:32:44.570 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:44 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:32:44.570 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:44 smithi105 ceph-mon[23707]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:44.570 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:44 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/274135378' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:32:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:44 smithi118 ceph-mon[25717]: pgmap v33: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-09-13T22:32:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:44 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:32:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:44 smithi118 ceph-mon[25717]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:32:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:44 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/274135378' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:32:45.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 81286 10.53 MB/sec execute 73 sec latency 46.774 ms 2024-09-13T22:32:46.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 81526 10.48 MB/sec execute 74 sec latency 76.276 ms 2024-09-13T22:32:46.635 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:46 smithi105 ceph-mon[23707]: pgmap v34: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 714 op/s 2024-09-13T22:32:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:46 smithi118 ceph-mon[25717]: pgmap v34: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 714 op/s 2024-09-13T22:32:47.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 81801 10.39 MB/sec execute 75 sec latency 65.244 ms 2024-09-13T22:32:48.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 82139 10.35 MB/sec execute 76 sec latency 50.966 ms 2024-09-13T22:32:48.684 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:48 smithi105 ceph-mon[23707]: pgmap v35: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 488 op/s 2024-09-13T22:32:48.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:48 smithi118 ceph-mon[25717]: pgmap v35: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 488 op/s 2024-09-13T22:32:49.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 82455 10.33 MB/sec execute 77 sec latency 50.923 ms 2024-09-13T22:32:50.151 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:50 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:50.152 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:50 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:50.152 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:50 smithi105 ceph-mon[23707]: Reconfiguring node-exporter.smithi118 (dependencies changed)... 2024-09-13T22:32:50.152 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:50 smithi105 ceph-mon[23707]: Deploying daemon node-exporter.smithi118 on smithi118 2024-09-13T22:32:50.152 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:50 smithi105 ceph-mon[23707]: pgmap v36: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 488 op/s 2024-09-13T22:32:50.302 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:50 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:50.303 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:50 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:50.303 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:50 smithi118 ceph-mon[25717]: Reconfiguring node-exporter.smithi118 (dependencies changed)... 2024-09-13T22:32:50.303 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:50 smithi118 ceph-mon[25717]: Deploying daemon node-exporter.smithi118 on smithi118 2024-09-13T22:32:50.303 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:50 smithi118 ceph-mon[25717]: pgmap v36: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 488 op/s 2024-09-13T22:32:50.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 82870 10.35 MB/sec execute 78 sec latency 32.505 ms 2024-09-13T22:32:51.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 83435 10.33 MB/sec execute 79 sec latency 34.914 ms 2024-09-13T22:32:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:52 smithi118 ceph-mon[25717]: pgmap v37: 97 pgs: 97 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 679 op/s 2024-09-13T22:32:52.492 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:52 smithi105 ceph-mon[23707]: pgmap v37: 97 pgs: 97 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 679 op/s 2024-09-13T22:32:52.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 83878 10.34 MB/sec execute 80 sec latency 59.622 ms 2024-09-13T22:32:53.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 84259 10.41 MB/sec execute 81 sec latency 65.943 ms 2024-09-13T22:32:54.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 84703 10.52 MB/sec execute 82 sec latency 43.259 ms 2024-09-13T22:32:54.552 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:54 smithi105 ceph-mon[23707]: pgmap v38: 97 pgs: 97 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 480 op/s 2024-09-13T22:32:54.702 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:54 smithi118 ceph-mon[25717]: pgmap v38: 97 pgs: 97 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 480 op/s 2024-09-13T22:32:55.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 85022 10.46 MB/sec execute 83 sec latency 58.128 ms 2024-09-13T22:32:55.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:55.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:55.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:32:55.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:32:55.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi105.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi105.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:32:55.832 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:55.833 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:55 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:32:56.059 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:56.059 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:56.059 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi105.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi105.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:32:56.060 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T22:32:56.061 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:56.061 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:55 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:32:56.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 85359 10.42 MB/sec execute 84 sec latency 45.296 ms 2024-09-13T22:32:56.848 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:56 smithi105 ceph-mon[23707]: Upgrade: Need to upgrade myself (mgr.smithi118.rbhtgz) 2024-09-13T22:32:56.848 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:56 smithi105 ceph-mon[23707]: Upgrade: Need to upgrade myself (mgr.smithi118.rbhtgz) 2024-09-13T22:32:56.848 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:56 smithi105 ceph-mon[23707]: pgmap v39: 97 pgs: 97 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 742 op/s 2024-09-13T22:32:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:56 smithi118 ceph-mon[25717]: Upgrade: Need to upgrade myself (mgr.smithi118.rbhtgz) 2024-09-13T22:32:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:56 smithi118 ceph-mon[25717]: Upgrade: Need to upgrade myself (mgr.smithi118.rbhtgz) 2024-09-13T22:32:57.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:56 smithi118 ceph-mon[25717]: pgmap v39: 97 pgs: 97 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 742 op/s 2024-09-13T22:32:57.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 85675 10.38 MB/sec execute 85 sec latency 92.255 ms 2024-09-13T22:32:58.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 86006 10.35 MB/sec execute 86 sec latency 63.794 ms 2024-09-13T22:32:58.651 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:58 smithi105 ceph-mon[23707]: pgmap v40: 97 pgs: 97 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-13T22:32:58.651 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:58 smithi105 ceph-mon[23707]: Upgrade: Updating mgr.smithi105.swmmea 2024-09-13T22:32:58.651 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:58 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:58.651 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:58 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi105.swmmea", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:32:58.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:58 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi105.swmmea", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:32:58.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:58 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:32:58.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:58 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:32:58.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:58 smithi105 ceph-mon[23707]: Deploying daemon mgr.smithi105.swmmea on smithi105 2024-09-13T22:32:58.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:32:58 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:32:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:58 smithi118 ceph-mon[25717]: pgmap v40: 97 pgs: 97 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-13T22:32:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:58 smithi118 ceph-mon[25717]: Upgrade: Updating mgr.smithi105.swmmea 2024-09-13T22:32:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:58 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:32:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:58 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi105.swmmea", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:32:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:58 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi105.swmmea", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:32:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:58 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:32:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:58 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:32:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:58 smithi118 ceph-mon[25717]: Deploying daemon mgr.smithi105.swmmea on smithi105 2024-09-13T22:32:59.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:32:58 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:32:59.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 86380 10.34 MB/sec execute 87 sec latency 75.464 ms 2024-09-13T22:33:00.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 86825 10.34 MB/sec execute 88 sec latency 49.821 ms 2024-09-13T22:33:00.664 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:00 smithi105 ceph-mon[23707]: pgmap v41: 97 pgs: 97 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-13T22:33:01.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:00 smithi118 ceph-mon[25717]: pgmap v41: 97 pgs: 97 active+clean; 347 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-13T22:33:01.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 87217 10.31 MB/sec execute 89 sec latency 60.586 ms 2024-09-13T22:33:02.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 87618 10.34 MB/sec execute 90 sec latency 67.432 ms 2024-09-13T22:33:02.718 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:02 smithi105 ceph-mon[23707]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 665 op/s 2024-09-13T22:33:03.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:02 smithi118 ceph-mon[25717]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 665 op/s 2024-09-13T22:33:03.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 87996 10.39 MB/sec execute 91 sec latency 55.953 ms 2024-09-13T22:33:04.497 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:04 smithi105 ceph-mon[23707]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 474 op/s 2024-09-13T22:33:04.497 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:04 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:04.497 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:04 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:04.497 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:04 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:04.497 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:04 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:04.498 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:04 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:04.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 88375 10.44 MB/sec execute 92 sec latency 57.856 ms 2024-09-13T22:33:04.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:04 smithi118 ceph-mon[25717]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 474 op/s 2024-09-13T22:33:04.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:04 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:04.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:04 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:04.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:04 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:04.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:04 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:04.733 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:04 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:05.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 88612 10.40 MB/sec execute 93 sec latency 95.225 ms 2024-09-13T22:33:06.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 88826 10.32 MB/sec execute 94 sec latency 70.792 ms 2024-09-13T22:33:06.632 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:06 smithi105 ceph-mon[23707]: pgmap v44: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 703 op/s 2024-09-13T22:33:06.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:06 smithi118 ceph-mon[25717]: pgmap v44: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 703 op/s 2024-09-13T22:33:07.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 89074 10.24 MB/sec execute 95 sec latency 85.998 ms 2024-09-13T22:33:08.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 89431 10.22 MB/sec execute 96 sec latency 57.290 ms 2024-09-13T22:33:08.690 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:08 smithi105 ceph-mon[23707]: pgmap v45: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-13T22:33:08.690 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:08 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:08.690 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:08 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:08 smithi118 ceph-mon[25717]: pgmap v45: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-13T22:33:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:08 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:09.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:08 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:09.524 INFO:teuthology.orchestra.run.smithi105.stdout: 5 89901 10.29 MB/sec execute 97 sec latency 78.969 ms 2024-09-13T22:33:10.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 90293 10.25 MB/sec execute 98 sec latency 59.717 ms 2024-09-13T22:33:10.749 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:10 smithi105 ceph-mon[23707]: pgmap v46: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-13T22:33:10.749 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:10 smithi105 ceph-mon[23707]: Standby manager daemon smithi105.swmmea restarted 2024-09-13T22:33:10.749 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:10 smithi105 ceph-mon[23707]: Standby manager daemon smithi105.swmmea started 2024-09-13T22:33:10.749 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:10 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi105.swmmea/crt"}]: dispatch 2024-09-13T22:33:10.749 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:10 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:33:10.749 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:10 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi105.swmmea/key"}]: dispatch 2024-09-13T22:33:10.749 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:10 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:33:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:10 smithi118 ceph-mon[25717]: pgmap v46: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-13T22:33:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:10 smithi118 ceph-mon[25717]: Standby manager daemon smithi105.swmmea restarted 2024-09-13T22:33:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:10 smithi118 ceph-mon[25717]: Standby manager daemon smithi105.swmmea started 2024-09-13T22:33:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:10 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi105.swmmea/crt"}]: dispatch 2024-09-13T22:33:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:10 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:33:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:10 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi105.swmmea/key"}]: dispatch 2024-09-13T22:33:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:10 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:33:11.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 90661 10.20 MB/sec execute 99 sec latency 50.280 ms 2024-09-13T22:33:11.758 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:11 smithi105 ceph-mon[23707]: mgrmap e27: smithi118.rbhtgz(active, since 88s), standbys: smithi105.swmmea 2024-09-13T22:33:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:11 smithi118 ceph-mon[25717]: mgrmap e27: smithi118.rbhtgz(active, since 88s), standbys: smithi105.swmmea 2024-09-13T22:33:12.533 INFO:teuthology.orchestra.run.smithi105.stdout: 5 91370 10.32 MB/sec execute 100 sec latency 45.416 ms 2024-09-13T22:33:12.770 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:12 smithi105 ceph-mon[23707]: pgmap v47: 97 pgs: 97 active+clean; 346 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.4 MiB/s wr, 630 op/s 2024-09-13T22:33:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:12 smithi118 ceph-mon[25717]: pgmap v47: 97 pgs: 97 active+clean; 346 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.4 MiB/s wr, 630 op/s 2024-09-13T22:33:13.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 91773 10.40 MB/sec execute 101 sec latency 69.396 ms 2024-09-13T22:33:14.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 92131 10.41 MB/sec execute 102 sec latency 49.543 ms 2024-09-13T22:33:14.555 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:14 smithi105 ceph-mon[23707]: pgmap v48: 97 pgs: 97 active+clean; 346 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-13T22:33:14.555 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:14 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:33:14.555 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:14 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:14.555 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:14 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:14.819 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:33:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:14 smithi118 ceph-mon[25717]: pgmap v48: 97 pgs: 97 active+clean; 346 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-13T22:33:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:14 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:33:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:14 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:14 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:15.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 92423 10.35 MB/sec execute 103 sec latency 72.965 ms 2024-09-13T22:33:15.833 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:15 smithi105 ceph-mon[23707]: mgrmap e28: smithi118.rbhtgz(active, since 92s), standbys: smithi105.swmmea 2024-09-13T22:33:15.833 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:15 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:15.833 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:15 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:15.840 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:33:15.840 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (65s) 1s ago 10m 19.4M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:33:15.840 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (10m) 1s ago 10m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:33:15.840 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (9m) 8s ago 9m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:33:15.840 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (57s) 1s ago 10m 44.6M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:33:15.840 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (4m) 1s ago 4m 36.7M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:33:15.840 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (4m) 8s ago 4m 13.4M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (13s) 1s ago 11m 90.3M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283 running (118s) 8s ago 9m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 25b0284f8ebd 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (11m) 1s ago 11m 52.5M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (9m) 8s ago 9m 40.5M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (3m) 1s ago 3m 245M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (3m) 8s ago 3m 78.6M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (50s) 1s ago 9m 8300k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (21s) 8s ago 9m 7801k - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (8m) 1s ago 8m 383M 1936M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (8m) 1s ago 8m 270M 1936M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (7m) 1s ago 7m 235M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:33:15.841 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (7m) 1s ago 7m 336M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:33:15.842 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (6m) 8s ago 6m 344M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:33:15.842 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (6m) 8s ago 6m 342M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:33:15.842 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (6m) 8s ago 6m 288M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:33:15.842 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (5m) 8s ago 5m 452M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:33:15.842 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (27s) 1s ago 9m 43.9M - 2.43.0 a07b618ecd1d a437725a9f01 2024-09-13T22:33:16.059 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:15 smithi118 ceph-mon[25717]: mgrmap e28: smithi118.rbhtgz(active, since 92s), standbys: smithi105.swmmea 2024-09-13T22:33:16.059 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:15 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:16.059 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:15 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:16.439 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:33:16.439 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:33:16.439 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:33:16.439 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:33:16.439 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:33:16.439 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:33:16.439 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:33:16.439 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:33:16.440 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:33:16.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 92694 10.28 MB/sec execute 104 sec latency 40.415 ms 2024-09-13T22:33:16.842 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:16 smithi105 ceph-mon[23707]: from='client.24513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:16.842 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:16 smithi105 ceph-mon[23707]: pgmap v49: 97 pgs: 97 active+clean; 350 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.1 MiB/s wr, 681 op/s 2024-09-13T22:33:16.842 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:16 smithi105 ceph-mon[23707]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:16.842 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:16 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/1102390736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:33:16.934 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:33:16.934 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:33:16.934 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:33:16.934 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:33:16.935 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:33:16.935 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:33:16.935 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:33:16.935 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "2/23 daemons upgraded", 2024-09-13T22:33:16.935 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading mgr daemons", 2024-09-13T22:33:16.935 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:33:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:33:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:16 smithi118 ceph-mon[25717]: from='client.24513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:16 smithi118 ceph-mon[25717]: pgmap v49: 97 pgs: 97 active+clean; 350 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.1 MiB/s wr, 681 op/s 2024-09-13T22:33:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:16 smithi118 ceph-mon[25717]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:16 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/1102390736' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:33:17.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 93051 10.27 MB/sec execute 105 sec latency 39.678 ms 2024-09-13T22:33:17.532 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:33:17.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:17 smithi105 ceph-mon[23707]: from='client.24517 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:17.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:17 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/4181682826' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:33:18.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:17 smithi118 ceph-mon[25717]: from='client.24517 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:18.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:17 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/4181682826' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:33:18.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 93477 10.32 MB/sec execute 106 sec latency 35.992 ms 2024-09-13T22:33:18.880 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:18 smithi105 ceph-mon[23707]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:18.880 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:18 smithi105 ceph-mon[23707]: pgmap v50: 97 pgs: 97 active+clean; 350 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-13T22:33:18.998 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:18 smithi118 ceph-mon[25717]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:18.998 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:18 smithi118 ceph-mon[25717]: pgmap v50: 97 pgs: 97 active+clean; 350 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-13T22:33:19.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 93918 10.29 MB/sec execute 107 sec latency 41.834 ms 2024-09-13T22:33:20.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 94241 10.24 MB/sec execute 108 sec latency 59.064 ms 2024-09-13T22:33:20.911 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:20 smithi105 ceph-mon[23707]: pgmap v51: 97 pgs: 97 active+clean; 350 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-13T22:33:21.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:20 smithi118 ceph-mon[25717]: pgmap v51: 97 pgs: 97 active+clean; 350 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-13T22:33:21.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 94547 10.25 MB/sec execute 109 sec latency 70.210 ms 2024-09-13T22:33:22.507 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: pgmap v52: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 669 op/s 2024-09-13T22:33:22.507 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:22.507 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:22.507 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:33:22.507 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: Upgrade: Need to upgrade myself (mgr.smithi118.rbhtgz) 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: Upgrade: Need to upgrade myself (mgr.smithi118.rbhtgz) 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: Failing over to other MGR 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr fail", "who": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr fail", "who": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T22:33:22.508 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:22 smithi105 ceph-mon[23707]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:33:22.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 94842 10.24 MB/sec execute 110 sec latency 58.501 ms 2024-09-13T22:33:22.668 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: pgmap v52: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 669 op/s 2024-09-13T22:33:22.669 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:22.669 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:22.669 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:33:22.669 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:33:22.669 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:22.669 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' 2024-09-13T22:33:22.669 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:22.670 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: Upgrade: Need to upgrade myself (mgr.smithi118.rbhtgz) 2024-09-13T22:33:22.670 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: Upgrade: Need to upgrade myself (mgr.smithi118.rbhtgz) 2024-09-13T22:33:22.670 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: Failing over to other MGR 2024-09-13T22:33:22.670 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: from='mgr.24427 172.21.15.118:0/2285212531' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr fail", "who": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:33:22.670 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "mgr fail", "who": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:33:22.670 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T22:33:22.670 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:22 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:33:23.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 95204 10.29 MB/sec execute 111 sec latency 51.881 ms 2024-09-13T22:33:23.529 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd='[{"prefix": "mgr fail", "who": "smithi118.rbhtgz"}]': finished 2024-09-13T22:33:23.529 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: mgrmap e29: smithi105.swmmea(active, starting, since 0.855684s), standbys: smithi118.rbhtgz 2024-09-13T22:33:23.529 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:33:23.529 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:33:23.529 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:33:23.529 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:33:23.529 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:33:23.529 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rbhtgz", "id": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:33:23.529 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: Manager daemon smithi105.swmmea is now available 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:33:23.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:33:23.531 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:23 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:33:23.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.24427 ' entity='mgr.smithi118.rbhtgz' cmd='[{"prefix": "mgr fail", "who": "smithi118.rbhtgz"}]': finished 2024-09-13T22:33:23.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: mgrmap e29: smithi105.swmmea(active, starting, since 0.855684s), standbys: smithi118.rbhtgz 2024-09-13T22:33:23.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:33:23.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:33:23.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rbhtgz", "id": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:33:23.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:33:23.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:33:23.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: Manager daemon smithi105.swmmea is now available 2024-09-13T22:33:23.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:23.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:33:23.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:33:23.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:23 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:33:24.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 95544 10.30 MB/sec execute 112 sec latency 41.871 ms 2024-09-13T22:33:24.545 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:24 smithi105 ceph-mon[23707]: mgrmap e30: smithi105.swmmea(active, since 1.88239s), standbys: smithi118.rbhtgz 2024-09-13T22:33:24.545 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:24 smithi105 ceph-mon[23707]: pgmap v3: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T22:33:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:24 smithi118 ceph-mon[25717]: mgrmap e30: smithi105.swmmea(active, since 1.88239s), standbys: smithi118.rbhtgz 2024-09-13T22:33:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:24 smithi118 ceph-mon[25717]: pgmap v3: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T22:33:25.525 INFO:teuthology.orchestra.run.smithi105.stdout: 5 95844 10.25 MB/sec execute 113 sec latency 43.868 ms 2024-09-13T22:33:25.577 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:25 smithi105 ceph-mon[23707]: mgrmap e31: smithi105.swmmea(active, since 2s), standbys: smithi118.rbhtgz 2024-09-13T22:33:25.577 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:25 smithi105 ceph-mon[23707]: [13/Sep/2024:22:33:25] ENGINE Bus STARTING 2024-09-13T22:33:25.577 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:25 smithi105 ceph-mon[23707]: [13/Sep/2024:22:33:25] ENGINE Serving on https://172.21.15.105:7150 2024-09-13T22:33:25.577 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:25 smithi105 ceph-mon[23707]: [13/Sep/2024:22:33:25] ENGINE Error in HTTPServer.serve 2024-09-13T22:33:25.577 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Traceback (most recent call last): 2024-09-13T22:33:25.577 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:33:25.577 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:33:25.577 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:33:25.577 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self._run(expiration_interval) 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: s = self.context.wrap_socket( 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: return self.sslsocket_class._create( 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self.do_handshake() 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self._sslobj.do_handshake() 2024-09-13T22:33:25.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:33:25.814 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:25 smithi118 ceph-mon[25717]: mgrmap e31: smithi105.swmmea(active, since 2s), standbys: smithi118.rbhtgz 2024-09-13T22:33:25.814 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:25 smithi118 ceph-mon[25717]: [13/Sep/2024:22:33:25] ENGINE Bus STARTING 2024-09-13T22:33:25.814 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:25 smithi118 ceph-mon[25717]: [13/Sep/2024:22:33:25] ENGINE Serving on https://172.21.15.105:7150 2024-09-13T22:33:25.814 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:25 smithi118 ceph-mon[25717]: [13/Sep/2024:22:33:25] ENGINE Error in HTTPServer.serve 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Traceback (most recent call last): 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._run(expiration_interval) 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s = self.context.wrap_socket( 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: return self.sslsocket_class._create( 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:33:25.815 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self.do_handshake() 2024-09-13T22:33:25.816 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:33:25.816 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._sslobj.do_handshake() 2024-09-13T22:33:25.816 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:33:26.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 96068 10.20 MB/sec execute 114 sec latency 88.631 ms 2024-09-13T22:33:26.633 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:26 smithi105 ceph-mon[23707]: [13/Sep/2024:22:33:25] ENGINE Serving on http://172.21.15.105:8765 2024-09-13T22:33:26.633 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:26 smithi105 ceph-mon[23707]: [13/Sep/2024:22:33:25] ENGINE Bus STARTED 2024-09-13T22:33:26.843 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:26 smithi118 ceph-mon[25717]: [13/Sep/2024:22:33:25] ENGINE Serving on http://172.21.15.105:8765 2024-09-13T22:33:26.843 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:26 smithi118 ceph-mon[25717]: [13/Sep/2024:22:33:25] ENGINE Bus STARTED 2024-09-13T22:33:27.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 96368 10.17 MB/sec execute 115 sec latency 46.533 ms 2024-09-13T22:33:27.658 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:27 smithi105 ceph-mon[23707]: pgmap v5: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T22:33:27.658 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:27 smithi105 ceph-mon[23707]: mgrmap e32: smithi105.swmmea(active, since 4s), standbys: smithi118.rbhtgz 2024-09-13T22:33:27.658 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:27 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:27.658 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:27 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:27.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:27 smithi118 ceph-mon[25717]: pgmap v5: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T22:33:27.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:27 smithi118 ceph-mon[25717]: mgrmap e32: smithi105.swmmea(active, since 4s), standbys: smithi118.rbhtgz 2024-09-13T22:33:27.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:27 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:27.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:27 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 96585 10.13 MB/sec execute 116 sec latency 50.304 ms 2024-09-13T22:33:29.196 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:29 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:29.196 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:29 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:29.196 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:29 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/3768951836' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:33:29.196 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:29 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/3768951836' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:33:29.196 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:29 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/3768951836' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:33:29.196 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:29 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/3768951836' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:33:29.196 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:29 smithi105 ceph-mon[23707]: Standby manager daemon smithi118.rbhtgz restarted 2024-09-13T22:33:29.197 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:29 smithi105 ceph-mon[23707]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:33:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:29 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:29 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:29 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/3768951836' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:33:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:29 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/3768951836' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:33:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:29 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/3768951836' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:33:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:29 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/3768951836' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:33:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:29 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz restarted 2024-09-13T22:33:29.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:29 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:33:29.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 96812 10.10 MB/sec execute 117 sec latency 78.664 ms 2024-09-13T22:33:30.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:30 smithi105 ceph-mon[23707]: pgmap v6: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T22:33:30.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:30 smithi105 ceph-mon[23707]: mgrmap e33: smithi105.swmmea(active, since 7s), standbys: smithi118.rbhtgz 2024-09-13T22:33:30.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 97154 10.09 MB/sec execute 118 sec latency 46.022 ms 2024-09-13T22:33:30.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:30 smithi118 ceph-mon[25717]: pgmap v6: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T22:33:30.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:30 smithi118 ceph-mon[25717]: mgrmap e33: smithi105.swmmea(active, since 7s), standbys: smithi118.rbhtgz 2024-09-13T22:33:31.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 97438 10.06 MB/sec execute 119 sec latency 52.800 ms 2024-09-13T22:33:32.329 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:32 smithi105 ceph-mon[23707]: pgmap v7: 97 pgs: 97 active+clean; 346 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 3.1 MiB/s wr, 324 op/s 2024-09-13T22:33:32.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 97680 9.99 MB/sec execute 120 sec latency 61.877 ms 2024-09-13T22:33:32.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:32 smithi118 ceph-mon[25717]: pgmap v7: 97 pgs: 97 active+clean; 346 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 3.1 MiB/s wr, 324 op/s 2024-09-13T22:33:33.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 98077 10.02 MB/sec execute 121 sec latency 43.941 ms 2024-09-13T22:33:33.604 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:33 smithi105 ceph-mon[23707]: pgmap v8: 97 pgs: 97 active+clean; 346 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.4 MiB/s wr, 252 op/s 2024-09-13T22:33:33.604 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:33 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:33.604 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:33 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:33.604 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:33 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:33.604 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:33 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:33.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:33 smithi118 ceph-mon[25717]: pgmap v8: 97 pgs: 97 active+clean; 346 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.4 MiB/s wr, 252 op/s 2024-09-13T22:33:33.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:33 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:33.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:33 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:33.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:33 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:33.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:33 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:34.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 98368 10.02 MB/sec execute 122 sec latency 49.006 ms 2024-09-13T22:33:35.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 98680 10.04 MB/sec execute 123 sec latency 57.962 ms 2024-09-13T22:33:35.684 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:35 smithi105 ceph-mon[23707]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.8 MiB/s wr, 398 op/s 2024-09-13T22:33:35.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:35 smithi118 ceph-mon[25717]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.8 MiB/s wr, 398 op/s 2024-09-13T22:33:36.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 99020 10.06 MB/sec execute 124 sec latency 53.985 ms 2024-09-13T22:33:37.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 99341 10.03 MB/sec execute 125 sec latency 67.282 ms 2024-09-13T22:33:37.719 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:37 smithi105 ceph-mon[23707]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.4 MiB/s wr, 362 op/s 2024-09-13T22:33:37.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:37 smithi118 ceph-mon[25717]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.4 MiB/s wr, 362 op/s 2024-09-13T22:33:38.491 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:38 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:33:38.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 99620 9.98 MB/sec execute 126 sec latency 53.070 ms 2024-09-13T22:33:38.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:38 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:33:39.515 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:39 smithi105 ceph-mon[23707]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.4 MiB/s wr, 363 op/s 2024-09-13T22:33:39.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 99897 9.97 MB/sec execute 127 sec latency 47.822 ms 2024-09-13T22:33:39.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:39 smithi118 ceph-mon[25717]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.4 MiB/s wr, 363 op/s 2024-09-13T22:33:40.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 100251 9.97 MB/sec execute 128 sec latency 37.307 ms 2024-09-13T22:33:40.662 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:40 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:40.662 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:40 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:40.662 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:40 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:33:40.663 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:40 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:33:40.663 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:40 smithi105 ceph-mon[23707]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:33:40.663 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:40 smithi105 ceph-mon[23707]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-13T22:33:40.663 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:40 smithi105 ceph-mon[23707]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:33:40.663 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:40 smithi105 ceph-mon[23707]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:33:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:40 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:40 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:40 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:33:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:40 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:33:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:40 smithi118 ceph-mon[25717]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:33:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:40 smithi118 ceph-mon[25717]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-13T22:33:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:40 smithi118 ceph-mon[25717]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:33:41.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:40 smithi118 ceph-mon[25717]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:33:41.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 100657 9.96 MB/sec execute 129 sec latency 42.771 ms 2024-09-13T22:33:41.949 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:33:41.949 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:33:41.949 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: pgmap v12: 97 pgs: 97 active+clean; 346 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.5 MiB/s wr, 548 op/s 2024-09-13T22:33:41.949 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:33:41.949 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:33:41.950 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:41.950 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:41.950 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:41.950 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:41.950 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:41.950 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:41.950 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:41 smithi105 ceph-mon[23707]: Reconfiguring prometheus.smithi105 (dependencies changed)... 2024-09-13T22:33:42.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:33:42.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:33:42.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: pgmap v12: 97 pgs: 97 active+clean; 346 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.5 MiB/s wr, 548 op/s 2024-09-13T22:33:42.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:33:42.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:33:42.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:42.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:42.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:42.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:42.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:42.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:42.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:41 smithi118 ceph-mon[25717]: Reconfiguring prometheus.smithi105 (dependencies changed)... 2024-09-13T22:33:42.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 100982 9.93 MB/sec execute 130 sec latency 48.137 ms 2024-09-13T22:33:42.956 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:42 smithi105 ceph-mon[23707]: Reconfiguring daemon prometheus.smithi105 on smithi105 2024-09-13T22:33:43.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:42 smithi118 ceph-mon[25717]: Reconfiguring daemon prometheus.smithi105 on smithi105 2024-09-13T22:33:43.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 101303 9.90 MB/sec execute 131 sec latency 60.265 ms 2024-09-13T22:33:44.004 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:43 smithi105 ceph-mon[23707]: pgmap v13: 97 pgs: 97 active+clean; 346 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 360 op/s 2024-09-13T22:33:44.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:43 smithi118 ceph-mon[25717]: pgmap v13: 97 pgs: 97 active+clean; 346 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 360 op/s 2024-09-13T22:33:44.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 101635 9.91 MB/sec execute 132 sec latency 54.817 ms 2024-09-13T22:33:45.526 INFO:teuthology.orchestra.run.smithi105.stdout: 5 101977 9.92 MB/sec execute 133 sec latency 34.394 ms 2024-09-13T22:33:46.041 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:45 smithi105 ceph-mon[23707]: pgmap v14: 97 pgs: 97 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.6 MiB/s wr, 572 op/s 2024-09-13T22:33:46.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:45 smithi118 ceph-mon[25717]: pgmap v14: 97 pgs: 97 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.6 MiB/s wr, 572 op/s 2024-09-13T22:33:46.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 102378 9.97 MB/sec execute 134 sec latency 48.042 ms 2024-09-13T22:33:47.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 102729 9.97 MB/sec execute 135 sec latency 42.198 ms 2024-09-13T22:33:47.809 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:47 smithi105 ceph-mon[23707]: pgmap v15: 97 pgs: 97 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-09-13T22:33:48.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:47 smithi118 ceph-mon[25717]: pgmap v15: 97 pgs: 97 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-09-13T22:33:48.116 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:33:48.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 103105 9.95 MB/sec execute 136 sec latency 59.196 ms 2024-09-13T22:33:48.923 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:33:48.923 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (98s) 16s ago 10m 19.4M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:33:48.923 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (10m) 16s ago 10m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:33:48.923 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (10m) 21s ago 10m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:33:48.923 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (90s) 16s ago 10m 45.2M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (4m) 16s ago 4m 36.7M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (4m) 21s ago 4m 13.7M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (46s) 16s ago 12m 476M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283 running (2m) 21s ago 10m 114M - 19.1.1-324-gecab7be5 0652ff2b5d78 25b0284f8ebd 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (12m) 16s ago 12m 55.1M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (10m) 21s ago 10m 41.4M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (4m) 16s ago 4m 254M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (4m) 21s ago 4m 78.8M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (83s) 16s ago 10m 8409k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (54s) 21s ago 9m 8070k - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (9m) 16s ago 9m 404M 1936M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:33:48.924 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (8m) 16s ago 8m 286M 1936M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:33:48.925 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (8m) 16s ago 8m 245M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:33:48.925 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (7m) 16s ago 7m 362M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:33:48.925 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (7m) 21s ago 7m 355M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:33:48.925 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (7m) 21s ago 7m 379M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:33:48.925 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (6m) 21s ago 6m 324M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:33:48.925 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (6m) 21s ago 6m 469M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:33:48.925 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (60s) 16s ago 9m 46.5M - 2.43.0 a07b618ecd1d a437725a9f01 2024-09-13T22:33:49.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:48 smithi105 ceph-mon[23707]: from='client.24543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:49.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:48 smithi118 ceph-mon[25717]: from='client.24543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:49.444 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:33:49.444 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:33:49.445 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:33:49.446 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:33:49.446 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T22:33:49.446 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:33:49.446 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:33:49.446 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:33:49.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 103317 9.90 MB/sec execute 137 sec latency 45.131 ms 2024-09-13T22:33:49.835 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:49 smithi105 ceph-mon[23707]: pgmap v16: 97 pgs: 97 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-09-13T22:33:49.835 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:49 smithi105 ceph-mon[23707]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:49.835 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:49 smithi105 ceph-mon[23707]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:49.835 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:49 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/2310176276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:33:49.878 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:33:49.878 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:33:49.878 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:33:49.878 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:33:49.879 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:33:49.879 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:33:49.879 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:33:49.879 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "2/23 daemons upgraded", 2024-09-13T22:33:49.879 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "", 2024-09-13T22:33:49.879 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:33:49.879 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:33:50.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:49 smithi118 ceph-mon[25717]: pgmap v16: 97 pgs: 97 active+clean; 342 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-09-13T22:33:50.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:49 smithi118 ceph-mon[25717]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:50.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:49 smithi118 ceph-mon[25717]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:50.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:49 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2310176276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:33:50.417 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:33:50.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 103604 9.88 MB/sec execute 138 sec latency 54.753 ms 2024-09-13T22:33:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:50 smithi118 ceph-mon[25717]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:50 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:50 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:50 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:33:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:50 smithi118 ceph-mon[25717]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:33:51.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:50 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:51.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:50 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/4093604532' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:33:51.121 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:50 smithi105 ceph-mon[23707]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:33:51.121 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:50 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:51.121 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:50 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:51.121 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:50 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:33:51.121 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:50 smithi105 ceph-mon[23707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T22:33:51.121 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:50 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:51.121 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:50 smithi105 ceph-mon[23707]: from='client.? 172.21.15.105:0/4093604532' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:33:51.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 103900 9.88 MB/sec execute 139 sec latency 51.061 ms 2024-09-13T22:33:52.127 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:51 smithi105 ceph-mon[23707]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.4 MiB/s wr, 606 op/s 2024-09-13T22:33:52.127 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:51 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:33:52.127 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:51 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:33:52.127 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:51 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:33:52.278 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:51 smithi118 ceph-mon[25717]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.4 MiB/s wr, 606 op/s 2024-09-13T22:33:52.278 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:51 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:33:52.278 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:51 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:33:52.278 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:51 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:33:52.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 104353 9.87 MB/sec execute 140 sec latency 43.400 ms 2024-09-13T22:33:53.059 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:52 smithi118 ceph-mon[25717]: Upgrade: Updating mgr.smithi118.rbhtgz 2024-09-13T22:33:53.059 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:52 smithi118 ceph-mon[25717]: Deploying daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:33:53.135 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:52 smithi105 ceph-mon[23707]: Upgrade: Updating mgr.smithi118.rbhtgz 2024-09-13T22:33:53.135 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:52 smithi105 ceph-mon[23707]: Deploying daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:33:53.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 104737 9.86 MB/sec execute 141 sec latency 73.123 ms 2024-09-13T22:33:54.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:53 smithi118 ceph-mon[25717]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 421 op/s 2024-09-13T22:33:54.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:53 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:33:54.174 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:53 smithi105 ceph-mon[23707]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 421 op/s 2024-09-13T22:33:54.174 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:53 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:33:54.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 105076 9.87 MB/sec execute 142 sec latency 86.023 ms 2024-09-13T22:33:55.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 105443 9.88 MB/sec execute 143 sec latency 50.982 ms 2024-09-13T22:33:56.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:55 smithi105 ceph-mon[23707]: pgmap v19: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 624 op/s 2024-09-13T22:33:56.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:55 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:56.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:55 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:56.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:55 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:56.237 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:55 smithi118 ceph-mon[25717]: pgmap v19: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 624 op/s 2024-09-13T22:33:56.237 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:55 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:56.237 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:55 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:56.237 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:55 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:33:56.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 105759 9.91 MB/sec execute 144 sec latency 54.348 ms 2024-09-13T22:33:57.291 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:57 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:57.291 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:57 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:57.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 106010 9.90 MB/sec execute 145 sec latency 51.179 ms 2024-09-13T22:33:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:57 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:57 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:33:58.248 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:58 smithi118 ceph-mon[25717]: pgmap v20: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 412 op/s 2024-09-13T22:33:58.331 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:58 smithi105 ceph-mon[23707]: pgmap v20: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 412 op/s 2024-09-13T22:33:58.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 106220 9.87 MB/sec execute 146 sec latency 58.682 ms 2024-09-13T22:33:59.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 106469 9.84 MB/sec execute 147 sec latency 47.718 ms 2024-09-13T22:33:59.630 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:33:59 smithi105 ceph-mon[23707]: pgmap v21: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 412 op/s 2024-09-13T22:33:59.709 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:33:59 smithi118 ceph-mon[25717]: pgmap v21: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 412 op/s 2024-09-13T22:34:00.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 106682 9.79 MB/sec execute 148 sec latency 115.485 ms 2024-09-13T22:34:01.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 106848 9.74 MB/sec execute 149 sec latency 56.133 ms 2024-09-13T22:34:01.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: pgmap v22: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.3 MiB/s wr, 593 op/s 2024-09-13T22:34:01.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:01.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:01.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:01.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:01.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: Standby manager daemon smithi118.rbhtgz restarted 2024-09-13T22:34:01.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:34:01.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/2474648617' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:34:01.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/2474648617' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:34:01.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/2474648617' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:34:01.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:01 smithi105 ceph-mon[23707]: from='mgr.? 172.21.15.118:0/2474648617' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:34:02.073 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: pgmap v22: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.3 MiB/s wr, 593 op/s 2024-09-13T22:34:02.073 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:02.073 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:02.073 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:02.073 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:02.073 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz restarted 2024-09-13T22:34:02.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:34:02.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/2474648617' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:34:02.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/2474648617' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:34:02.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/2474648617' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:34:02.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:01 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/2474648617' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:34:02.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 107059 9.72 MB/sec execute 150 sec latency 60.045 ms 2024-09-13T22:34:03.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 107310 9.71 MB/sec execute 151 sec latency 61.358 ms 2024-09-13T22:34:03.819 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:03 smithi105 ceph-mon[23707]: pgmap v23: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 382 op/s 2024-09-13T22:34:03.819 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:03 smithi105 ceph-mon[23707]: mgrmap e34: smithi105.swmmea(active, since 40s), standbys: smithi118.rbhtgz 2024-09-13T22:34:03.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:03 smithi118 ceph-mon[25717]: pgmap v23: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 382 op/s 2024-09-13T22:34:03.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:03 smithi118 ceph-mon[25717]: mgrmap e34: smithi105.swmmea(active, since 40s), standbys: smithi118.rbhtgz 2024-09-13T22:34:04.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 107881 9.72 MB/sec execute 152 sec latency 36.535 ms 2024-09-13T22:34:05.527 INFO:teuthology.orchestra.run.smithi105.stdout: 5 108348 9.72 MB/sec execute 153 sec latency 37.560 ms 2024-09-13T22:34:06.084 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:05 smithi105 ceph-mon[23707]: pgmap v24: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-09-13T22:34:06.084 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:05 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.084 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:05 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.084 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:05 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:05 smithi118 ceph-mon[25717]: pgmap v24: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-09-13T22:34:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:05 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:05 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:05 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 108752 9.74 MB/sec execute 154 sec latency 40.078 ms 2024-09-13T22:34:06.867 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.867 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:06.867 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:34:06.867 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.867 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.867 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:34:06.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:34:06.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: Upgrade: Setting container_image for all mgr 2024-09-13T22:34:06.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:06.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi105.swmmea"}]: dispatch 2024-09-13T22:34:06.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi105.swmmea"}]': finished 2024-09-13T22:34:06.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rbhtgz"}]: dispatch 2024-09-13T22:34:06.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rbhtgz"}]': finished 2024-09-13T22:34:06.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:06 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:34:07.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:07.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:07.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:34:07.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:07.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:07.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:34:07.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:34:07.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: Upgrade: Setting container_image for all mgr 2024-09-13T22:34:07.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:07.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi105.swmmea"}]: dispatch 2024-09-13T22:34:07.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi105.swmmea"}]': finished 2024-09-13T22:34:07.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rbhtgz"}]: dispatch 2024-09-13T22:34:07.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rbhtgz"}]': finished 2024-09-13T22:34:07.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:06 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:34:07.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 109099 9.75 MB/sec execute 155 sec latency 42.485 ms 2024-09-13T22:34:08.132 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:08 smithi105 ceph-mon[23707]: pgmap v25: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 323 op/s 2024-09-13T22:34:08.132 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:08 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:34:08.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:08 smithi118 ceph-mon[25717]: pgmap v25: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 323 op/s 2024-09-13T22:34:08.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:08 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:34:08.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 109449 9.78 MB/sec execute 156 sec latency 50.513 ms 2024-09-13T22:34:09.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 109719 9.78 MB/sec execute 157 sec latency 42.817 ms 2024-09-13T22:34:09.668 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:09 smithi105 ceph-mon[23707]: pgmap v26: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 508 op/s 2024-09-13T22:34:09.668 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:09 smithi105 ceph-mon[23707]: Upgrade: Updating mon.smithi105 2024-09-13T22:34:09.668 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:09 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:09.668 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:09 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:34:09.668 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:09 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:34:09.668 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:09 smithi105 ceph-mon[23707]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:09.669 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:09 smithi105 ceph-mon[23707]: Deploying daemon mon.smithi105 on smithi105 2024-09-13T22:34:09.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:09 smithi118 ceph-mon[25717]: pgmap v26: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 508 op/s 2024-09-13T22:34:09.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:09 smithi118 ceph-mon[25717]: Upgrade: Updating mon.smithi105 2024-09-13T22:34:09.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:09 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:09.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:09 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:34:09.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:09 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:34:09.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:09 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:09.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:09 smithi118 ceph-mon[25717]: Deploying daemon mon.smithi105 on smithi105 2024-09-13T22:34:10.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 109964 9.75 MB/sec execute 158 sec latency 76.069 ms 2024-09-13T22:34:11.237 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:11 smithi105 systemd[1]: Stopping Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:34:11.490 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:11 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105[23702]: 2024-09-13T22:34:11.474+0000 7f4088802700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi105 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:34:11.490 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:11 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105[23702]: 2024-09-13T22:34:11.474+0000 7f4088802700 -1 mon.smithi105@0(leader) e2 *** Got Signal Terminated *** 2024-09-13T22:34:11.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 110210 9.71 MB/sec execute 159 sec latency 87.303 ms 2024-09-13T22:34:11.746 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:11 smithi105 podman[65388]: 2024-09-13 22:34:11.495511491 +0000 UTC m=+0.158576173 container died 3e80a4cbe1c04e08a5626d09c28f8701db8d1ce6dd67635876c8e3af74ac06da (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, RELEASE=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., architecture=x86_64, version=8, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , name=centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870) 2024-09-13T22:34:12.501 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:12 smithi105 podman[65388]: 2024-09-13 22:34:12.264948708 +0000 UTC m=+0.928013392 container cleanup 3e80a4cbe1c04e08a5626d09c28f8701db8d1ce6dd67635876c8e3af74ac06da (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, maintainer=Guillaume Abrioux , name=centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:34:12.501 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:12 smithi105 bash[65388]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105 2024-09-13T22:34:12.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 110390 9.67 MB/sec execute 160 sec latency 55.142 ms 2024-09-13T22:34:12.754 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:12 smithi105 podman[65401]: 2024-09-13 22:34:12.634980618 +0000 UTC m=+1.135395311 container remove 3e80a4cbe1c04e08a5626d09c28f8701db8d1ce6dd67635876c8e3af74ac06da (image=quay.io/ceph/ceph:v17.2.0, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, ceph=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.openshift.expose-services=, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, architecture=x86_64) 2024-09-13T22:34:13.022 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:12 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi105.service: Deactivated successfully. 2024-09-13T22:34:13.022 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:12 smithi105 systemd[1]: Stopped Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:34:13.022 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:12 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi105.service: Consumed 14.966s CPU time. 2024-09-13T22:34:13.273 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 systemd[1]: Starting Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:34:13.528 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 podman[65536]: 2024-09-13 22:34:13.393982525 +0000 UTC m=+0.019466067 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:34:13.528 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 podman[65536]: 2024-09-13 22:34:13.527770137 +0000 UTC m=+0.153253659 container create 5855b3e3149363c1d95a2e13c82fd5515d0cc5f65e215e7eff28ef2a6907052d (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_CLEAN=True) 2024-09-13T22:34:13.529 INFO:teuthology.orchestra.run.smithi105.stdout: 5 110592 9.65 MB/sec execute 161 sec latency 78.471 ms 2024-09-13T22:34:14.036 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 podman[65536]: 2024-09-13 22:34:13.915266725 +0000 UTC m=+0.540750255 container init 5855b3e3149363c1d95a2e13c82fd5515d0cc5f65e215e7eff28ef2a6907052d (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-13T22:34:14.036 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 podman[65536]: 2024-09-13 22:34:13.920011727 +0000 UTC m=+0.545495258 container start 5855b3e3149363c1d95a2e13c82fd5515d0cc5f65e215e7eff28ef2a6907052d (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:34:14.036 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:34:14.036 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T22:34:14.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: pidfile_write: ignore empty --pid-file 2024-09-13T22:34:14.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: load: jerasure load: lrc 2024-09-13T22:34:14.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: RocksDB version: 7.9.2 2024-09-13T22:34:14.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Git sha 0 2024-09-13T22:34:14.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T22:34:14.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: DB SUMMARY 2024-09-13T22:34:14.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: DB Session ID: JQI72WHOKF9IGHR2BO0A 2024-09-13T22:34:14.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:34:14.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:34:14.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-09-13T22:34:14.038 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi105/store.db dir, Total Num: 1, files: 000030.sst 2024-09-13T22:34:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi105/store.db: 000028.log size: 2674704 ; 2024-09-13T22:34:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:34:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:34:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:34:14.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T22:34:14.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:34:14.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T22:34:14.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.env: 0x55ed45321940 2024-09-13T22:34:14.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T22:34:14.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.info_log: 0x55ed479d78c0 2024-09-13T22:34:14.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:34:14.041 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.statistics: (nil) 2024-09-13T22:34:14.041 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.use_fsync: 0 2024-09-13T22:34:14.042 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:34:14.042 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:34:14.042 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:34:14.042 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:34:14.042 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:34:14.042 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:34:14.043 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:34:14.043 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:34:14.043 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:34:14.043 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:34:14.043 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:34:14.043 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.db_log_dir: 2024-09-13T22:34:14.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.wal_dir: 2024-09-13T22:34:14.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:34:14.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:34:14.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:34:14.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:34:14.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:34:14.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:34:14.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:34:14.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:34:14.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.write_buffer_manager: 0x55ed479e6dc0 2024-09-13T22:34:14.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:34:14.046 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:34:14.046 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:34:14.046 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:34:14.046 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:34:14.046 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:34:14.046 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:34:14.047 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:34:14.047 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.unordered_write: 0 2024-09-13T22:34:14.047 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:34:14.047 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:34:14.047 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:34:14.047 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:34:14.048 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.row_cache: None 2024-09-13T22:34:14.048 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.wal_filter: None 2024-09-13T22:34:14.050 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:34:14.050 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:34:14.050 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:34:14.050 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:34:14.050 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.wal_compression: 0 2024-09-13T22:34:14.050 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:34:14.051 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:34:14.051 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:34:14.051 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:34:14.051 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:34:14.051 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:34:14.052 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:34:14.052 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:34:14.052 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:34:14.052 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:34:14.052 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:34:14.052 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T22:34:14.053 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:34:14.053 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:34:14.053 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:34:14.053 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:34:14.053 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:34:14.053 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:34:14.054 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:34:14.054 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:34:14.054 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:34:14.054 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:34:14.054 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:34:14.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_open_files: -1 2024-09-13T22:34:14.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:34:14.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:34:14.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:34:14.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:34:14.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:34:14.056 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Compression algorithms supported: 2024-09-13T22:34:14.056 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: kZSTD supported: 0 2024-09-13T22:34:14.056 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:34:14.056 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:34:14.056 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:34:14.056 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:34:14.056 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:34:14.056 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:34:14.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:34:14.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:34:14.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T22:34:14.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi105/store.db/MANIFEST-000015 2024-09-13T22:34:14.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T22:34:14.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:34:14.057 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.merge_operator: 2024-09-13T22:34:14.058 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_filter: None 2024-09-13T22:34:14.058 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:34:14.058 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:34:14.058 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:34:14.058 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:34:14.058 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ed479d7520) 2024-09-13T22:34:14.058 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:34:14.058 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:34:14.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:34:14.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:34:14.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: index_type: 0 2024-09-13T22:34:14.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: data_block_index_type: 0 2024-09-13T22:34:14.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: index_shortening: 1 2024-09-13T22:34:14.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:34:14.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: checksum: 4 2024-09-13T22:34:14.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: no_block_cache: 0 2024-09-13T22:34:14.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_cache: 0x55ed479cb1f0 2024-09-13T22:34:14.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:34:14.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_cache_options: 2024-09-13T22:34:14.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: capacity : 536870912 2024-09-13T22:34:14.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: num_shard_bits : 4 2024-09-13T22:34:14.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: strict_capacity_limit : 0 2024-09-13T22:34:14.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:34:14.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_cache_compressed: (nil) 2024-09-13T22:34:14.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: persistent_cache: (nil) 2024-09-13T22:34:14.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_size: 4096 2024-09-13T22:34:14.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_size_deviation: 10 2024-09-13T22:34:14.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_restart_interval: 16 2024-09-13T22:34:14.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: index_block_restart_interval: 1 2024-09-13T22:34:14.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: metadata_block_size: 4096 2024-09-13T22:34:14.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: partition_filters: 0 2024-09-13T22:34:14.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: use_delta_encoding: 1 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: filter_policy: bloomfilter 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: whole_key_filtering: 1 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: verify_compression: 0 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: format_version: 5 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: enable_index_compression: 1 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: block_align: 0 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: max_auto_readahead_size: 262144 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: prepopulate_block_cache: 0 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: initial_auto_readahead_size: 8192 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:34:14.062 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression: NoCompression 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.num_levels: 7 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:34:14.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:34:14.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:34:14.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:34:14.066 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:34:14.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.ttl: 2592000 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:34:14.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi105/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11822, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a6aab0dc-3a27-43e8-a274-090fe047471b 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266853948406, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266853958721, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2475158, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11823, "largest_seqno": 12836, "table_properties": {"data_size": 2469835, "index_size": 3126, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1157, "raw_key_size": 10137, "raw_average_key_size": 23, "raw_value_size": 2460168, "raw_average_value_size": 5616, "num_data_blocks": 142, "num_entries": 438, "num_filter_entries": 438, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726266853, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a6aab0dc-3a27-43e8-a274-090fe047471b", "db_session_id": "JQI72WHOKF9IGHR2BO0A", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-13T22:34:14.069 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266853958870, "job": 1, "event": "recovery_finished"} 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi105/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ed479f8e00 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: DB pointer 0x55ed47af4000 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** DB Stats ** 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:34:14.070 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: L0 1/0 2.36 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 431.5 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: L6 1/0 6.84 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Sum 2/0 9.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 431.5 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 431.5 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:34:14.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 431.5 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T22:34:14.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:34:14.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T22:34:14.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:34:14.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:34:14.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-13T22:34:14.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:34:14.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:34:14.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:34:14.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:34:14.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Cumulative compaction: 0.00 GB write, 154.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:34:14.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Interval compaction: 0.00 GB write, 154.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:34:14.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:34:14.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Block cache BinnedLRUCache@0x55ed479cb1f0#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T22:34:14.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-09-13T22:34:14.074 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:34:14.074 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:34:14.074 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: starting mon.smithi105 rank 0 at public addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] at bind addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi105 fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:34:14.074 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: mon.smithi105@-1(???) e2 preinit fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:34:14.074 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: mon.smithi105@-1(???).mds e7 new map 2024-09-13T22:34:14.074 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: mon.smithi105@-1(???).mds e7 print_map 2024-09-13T22:34:14.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: e7 2024-09-13T22:34:14.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T22:34:14.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:34:14.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:34:14.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: legacy client fscid: 1 2024-09-13T22:34:14.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:34:14.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Filesystem 'foofs' (1) 2024-09-13T22:34:14.076 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: fs_name foofs 2024-09-13T22:34:14.076 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: epoch 5 2024-09-13T22:34:14.076 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T22:34:14.076 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: created 2024-09-13T22:28:57.698196+0000 2024-09-13T22:34:14.076 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: modified 2024-09-13T22:29:02.728668+0000 2024-09-13T22:34:14.076 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: tableserver 0 2024-09-13T22:34:14.077 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: root 0 2024-09-13T22:34:14.077 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: session_timeout 60 2024-09-13T22:34:14.077 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: session_autoclose 300 2024-09-13T22:34:14.077 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: max_file_size 1099511627776 2024-09-13T22:34:14.077 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: max_xattr_size 65536 2024-09-13T22:34:14.077 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: required_client_features {} 2024-09-13T22:34:14.078 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: last_failure 0 2024-09-13T22:34:14.078 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: last_failure_osd_epoch 0 2024-09-13T22:34:14.078 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:34:14.078 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: max_mds 1 2024-09-13T22:34:14.078 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: in 0 2024-09-13T22:34:14.078 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: up {0=14486} 2024-09-13T22:34:14.079 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: failed 2024-09-13T22:34:14.079 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: damaged 2024-09-13T22:34:14.079 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: stopped 2024-09-13T22:34:14.079 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: data_pools [3] 2024-09-13T22:34:14.079 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: metadata_pool 2 2024-09-13T22:34:14.079 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: inline_data disabled 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: balancer 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: bal_rank_mask -1 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: standby_count_wanted 1 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: qdb_cluster leader: 0 members: 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: [mds.foofs.smithi105.ddoxtb{0:14486} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.105:6834/1075044016,v1:172.21.15.105:6835/1075044016] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Standby daemons: 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: 2024-09-13T22:34:14.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: [mds.foofs.smithi118.izbofj{-1:24281} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T22:34:14.081 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: mon.smithi105@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T22:34:14.081 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: mon.smithi105@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:34:14.081 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: mon.smithi105@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:34:14.081 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: mon.smithi105@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:34:14.081 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:13 smithi105 ceph-mon[65554]: mon.smithi105@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-13T22:34:14.286 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 bash[65536]: 5855b3e3149363c1d95a2e13c82fd5515d0cc5f65e215e7eff28ef2a6907052d 2024-09-13T22:34:14.286 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 systemd[1]: Started Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:34:14.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 110883 9.64 MB/sec execute 162 sec latency 65.758 ms 2024-09-13T22:34:14.790 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:14.790 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:14.790 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: pgmap v28: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 389 op/s 2024-09-13T22:34:14.790 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: mon.smithi105 calling monitor election 2024-09-13T22:34:14.790 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: mon.smithi105 is new leader, mons smithi105,smithi118 in quorum (ranks 0,1) 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: monmap epoch 2 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: last_changed 2024-09-13T22:23:43.483561+0000 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: created 2024-09-13T22:21:27.083377+0000 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: min_mon_release 17 (quincy) 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: election_strategy: 1 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: 0: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.smithi105 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: 1: [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon.smithi118 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: mgrmap e34: smithi105.swmmea(active, since 53s), standbys: smithi118.rbhtgz 2024-09-13T22:34:14.791 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: overall HEALTH_OK 2024-09-13T22:34:14.792 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: from='mgr.14774 ' entity='' 2024-09-13T22:34:14.792 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:14 smithi105 ceph-mon[65554]: mgrmap e35: smithi105.swmmea(active, since 53s), standbys: smithi118.rbhtgz 2024-09-13T22:34:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: from='mgr.14774 172.21.15.105:0/749817973' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: pgmap v28: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 389 op/s 2024-09-13T22:34:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: mon.smithi105 calling monitor election 2024-09-13T22:34:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: mon.smithi105 is new leader, mons smithi105,smithi118 in quorum (ranks 0,1) 2024-09-13T22:34:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: monmap epoch 2 2024-09-13T22:34:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:34:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: last_changed 2024-09-13T22:23:43.483561+0000 2024-09-13T22:34:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: created 2024-09-13T22:21:27.083377+0000 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: min_mon_release 17 (quincy) 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: election_strategy: 1 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: 0: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.smithi105 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: 1: [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon.smithi118 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: mgrmap e34: smithi105.swmmea(active, since 53s), standbys: smithi118.rbhtgz 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: overall HEALTH_OK 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: from='mgr.14774 ' entity='' 2024-09-13T22:34:15.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:14 smithi118 ceph-mon[25717]: mgrmap e35: smithi105.swmmea(active, since 53s), standbys: smithi118.rbhtgz 2024-09-13T22:34:15.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 111304 9.63 MB/sec execute 163 sec latency 54.742 ms 2024-09-13T22:34:16.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 111689 9.63 MB/sec execute 164 sec latency 42.224 ms 2024-09-13T22:34:16.913 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:16 smithi105 ceph-mon[65554]: mgrmap e36: smithi105.swmmea(active, since 54s), standbys: smithi118.rbhtgz 2024-09-13T22:34:17.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:16 smithi118 ceph-mon[25717]: mgrmap e36: smithi105.swmmea(active, since 54s), standbys: smithi118.rbhtgz 2024-09-13T22:34:17.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 111994 9.60 MB/sec execute 165 sec latency 40.770 ms 2024-09-13T22:34:18.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 112316 9.62 MB/sec execute 166 sec latency 41.070 ms 2024-09-13T22:34:19.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 112746 9.67 MB/sec execute 167 sec latency 39.509 ms 2024-09-13T22:34:20.510 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:20 smithi105 ceph-mon[65554]: from='mgr.? 172.21.15.118:0/4212896975' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:34:20.510 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:20 smithi105 ceph-mon[65554]: from='mgr.? 172.21.15.118:0/4212896975' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:34:20.510 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:20 smithi105 ceph-mon[65554]: from='mgr.? 172.21.15.118:0/4212896975' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:34:20.510 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:20 smithi105 ceph-mon[65554]: from='mgr.? 172.21.15.118:0/4212896975' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:34:20.510 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:20 smithi105 ceph-mon[65554]: Standby manager daemon smithi118.rbhtgz restarted 2024-09-13T22:34:20.510 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:20 smithi105 ceph-mon[65554]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:34:20.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 113005 9.66 MB/sec execute 168 sec latency 78.563 ms 2024-09-13T22:34:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:20 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/4212896975' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/crt"}]: dispatch 2024-09-13T22:34:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:20 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/4212896975' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T22:34:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:20 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/4212896975' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rbhtgz/key"}]: dispatch 2024-09-13T22:34:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:20 smithi118 ceph-mon[25717]: from='mgr.? 172.21.15.118:0/4212896975' entity='mgr.smithi118.rbhtgz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T22:34:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:20 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz restarted 2024-09-13T22:34:20.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:20 smithi118 ceph-mon[25717]: Standby manager daemon smithi118.rbhtgz started 2024-09-13T22:34:21.528 INFO:teuthology.orchestra.run.smithi105.stdout: 5 113269 9.65 MB/sec execute 169 sec latency 94.012 ms 2024-09-13T22:34:21.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:21 smithi105 ceph-mon[65554]: mgrmap e37: smithi105.swmmea(active, since 58s), standbys: smithi118.rbhtgz 2024-09-13T22:34:21.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:21 smithi105 ceph-mon[65554]: Active manager daemon smithi105.swmmea restarted 2024-09-13T22:34:21.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:21 smithi105 ceph-mon[65554]: Activating manager daemon smithi105.swmmea 2024-09-13T22:34:21.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:21 smithi105 ceph-mon[65554]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T22:34:21.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:21 smithi105 ceph-mon[65554]: mgrmap e38: smithi105.swmmea(active, starting, since 0.0315153s), standbys: smithi118.rbhtgz 2024-09-13T22:34:21.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:21 smithi105 ceph-mon[65554]: from='mgr.34100 172.21.15.105:0/401911803' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:21.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:21 smithi105 ceph-mon[65554]: from='mgr.34100 172.21.15.105:0/401911803' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:21.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:21 smithi105 ceph-mon[65554]: from='mgr.34100 172.21.15.105:0/401911803' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:34:21.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:21 smithi105 ceph-mon[65554]: from='mgr.34100 172.21.15.105:0/401911803' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:34:21.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:21 smithi118 ceph-mon[25717]: mgrmap e37: smithi105.swmmea(active, since 58s), standbys: smithi118.rbhtgz 2024-09-13T22:34:21.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:21 smithi118 ceph-mon[25717]: Active manager daemon smithi105.swmmea restarted 2024-09-13T22:34:21.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:21 smithi118 ceph-mon[25717]: Activating manager daemon smithi105.swmmea 2024-09-13T22:34:21.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:21 smithi118 ceph-mon[25717]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T22:34:21.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:21 smithi118 ceph-mon[25717]: mgrmap e38: smithi105.swmmea(active, starting, since 0.0315153s), standbys: smithi118.rbhtgz 2024-09-13T22:34:21.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:21 smithi118 ceph-mon[25717]: from='mgr.34100 172.21.15.105:0/401911803' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:21.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:21 smithi118 ceph-mon[25717]: from='mgr.34100 172.21.15.105:0/401911803' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:21.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:21 smithi118 ceph-mon[25717]: from='mgr.34100 172.21.15.105:0/401911803' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:34:21.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:21 smithi118 ceph-mon[25717]: from='mgr.34100 172.21.15.105:0/401911803' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:34:22.537 INFO:teuthology.orchestra.run.smithi105.stdout: 5 113560 9.63 MB/sec execute 170 sec latency 67.560 ms 2024-09-13T22:34:23.539 INFO:teuthology.orchestra.run.smithi105.stdout: 5 113962 9.61 MB/sec execute 171 sec latency 28.010 ms 2024-09-13T22:34:24.547 INFO:teuthology.orchestra.run.smithi105.stdout: 5 114778 9.70 MB/sec execute 172 sec latency 33.875 ms 2024-09-13T22:34:25.547 INFO:teuthology.orchestra.run.smithi105.stdout: 5 115401 9.70 MB/sec execute 173 sec latency 47.213 ms 2024-09-13T22:34:26.547 INFO:teuthology.orchestra.run.smithi105.stdout: 5 116122 9.77 MB/sec execute 174 sec latency 28.288 ms 2024-09-13T22:34:27.547 INFO:teuthology.orchestra.run.smithi105.stdout: 5 116789 9.87 MB/sec execute 175 sec latency 28.586 ms 2024-09-13T22:34:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: 5 117241 9.86 MB/sec execute 176 sec latency 31.000 ms 2024-09-13T22:34:28.855 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: Active manager daemon smithi105.swmmea restarted 2024-09-13T22:34:28.855 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: Activating manager daemon smithi105.swmmea 2024-09-13T22:34:28.855 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T22:34:28.855 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: mgrmap e39: smithi105.swmmea(active, starting, since 0.00686869s), standbys: smithi118.rbhtgz 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rbhtgz", "id": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:34:28.856 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:34:28.857 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:34:28.857 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:34:28.857 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:34:28.857 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:34:28.857 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:34:28.857 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:28 smithi105 ceph-mon[65554]: Manager daemon smithi105.swmmea is now available 2024-09-13T22:34:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: Active manager daemon smithi105.swmmea restarted 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: Activating manager daemon smithi105.swmmea 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: mgrmap e39: smithi105.swmmea(active, starting, since 0.00686869s), standbys: smithi118.rbhtgz 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi105.swmmea", "id": "smithi105.swmmea"}]: dispatch 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rbhtgz", "id": "smithi118.rbhtgz"}]: dispatch 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:34:29.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:34:29.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:34:29.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:34:29.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:34:29.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:34:29.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:34:29.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T22:34:29.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T22:34:29.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T22:34:29.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:28 smithi118 ceph-mon[25717]: Manager daemon smithi105.swmmea is now available 2024-09-13T22:34:29.547 INFO:teuthology.orchestra.run.smithi105.stdout: 5 117538 9.83 MB/sec execute 177 sec latency 49.619 ms 2024-09-13T22:34:29.820 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:34:29.863 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:34:29.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:34:29.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:34:29.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:29 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:34:29.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:34:29.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:29 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:34:29.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:29 smithi105 ceph-mon[65554]: mgrmap e40: smithi105.swmmea(active, since 1.01286s), standbys: smithi118.rbhtgz 2024-09-13T22:34:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:29 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:34:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:29 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:34:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:29 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:34:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:29 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/mirror_snapshot_schedule"}]: dispatch 2024-09-13T22:34:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:29 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:34:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:29 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi105.swmmea/trash_purge_schedule"}]: dispatch 2024-09-13T22:34:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:29 smithi118 ceph-mon[25717]: mgrmap e40: smithi105.swmmea(active, since 1.01286s), standbys: smithi118.rbhtgz 2024-09-13T22:34:30.547 INFO:teuthology.orchestra.run.smithi105.stdout: 5 118059 9.86 MB/sec execute 178 sec latency 35.018 ms 2024-09-13T22:34:30.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:30 smithi105 ceph-mon[65554]: [13/Sep/2024:22:34:30] ENGINE Bus STARTING 2024-09-13T22:34:31.003 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:34:31.003 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (2m) 26s ago 11m 19.5M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:34:31.003 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (11m) 26s ago 11m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:34:31.003 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (10m) 30s ago 10m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:34:31.003 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (2m) 26s ago 11m 47.4M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (5m) 26s ago 5m 37.3M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (5m) 30s ago 5m 13.9M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (88s) 26s ago 12m 491M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (35s) 30s ago 10m 95.4M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (12m) 26s ago 12m 56.3M 2048M 17.2.0 e1d6a67b021e 3e80a4cbe1c0 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (10m) 30s ago 10m 42.0M 2048M 17.2.0 e1d6a67b021e 17b16ac8afb2 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (5m) 26s ago 5m 257M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (5m) 30s ago 5m 78.9M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (2m) 26s ago 11m 8174k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (96s) 30s ago 10m 8652k - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (10m) 26s ago 9m 443M 1936M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (9m) 26s ago 9m 320M 1936M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (9m) 26s ago 9m 272M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:34:31.004 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (8m) 26s ago 8m 397M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:34:31.005 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (8m) 30s ago 8m 380M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:34:31.005 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (7m) 30s ago 7m 412M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:34:31.005 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (7m) 30s ago 7m 347M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:34:31.005 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (7m) 30s ago 7m 498M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:34:31.005 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (41s) 26s ago 10m 43.7M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:34:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:30 smithi118 ceph-mon[25717]: [13/Sep/2024:22:34:30] ENGINE Bus STARTING 2024-09-13T22:34:31.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 118680 9.89 MB/sec execute 179 sec latency 59.152 ms 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:34:31.736 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:34:31.737 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:34:31.737 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:34:31.737 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:34:31.737 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:34:31.737 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:34:31.737 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-13T22:34:31.737 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T22:34:31.737 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:34:31.737 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:34:31.901 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:31 smithi105 ceph-mon[65554]: [13/Sep/2024:22:34:30] ENGINE Serving on https://172.21.15.105:7150 2024-09-13T22:34:31.901 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:31 smithi105 ceph-mon[65554]: [13/Sep/2024:22:34:30] ENGINE Error in HTTPServer.serve 2024-09-13T22:34:31.901 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: Traceback (most recent call last): 2024-09-13T22:34:31.901 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:34:31.901 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:34:31.901 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self._run(expiration_interval) 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: s = self.context.wrap_socket( 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: return self.sslsocket_class._create( 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self.do_handshake() 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:34:31.902 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: self._sslobj.do_handshake() 2024-09-13T22:34:31.903 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:34:31.903 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:31 smithi105 ceph-mon[65554]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:34:31.903 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:31 smithi105 ceph-mon[65554]: [13/Sep/2024:22:34:30] ENGINE Serving on http://172.21.15.105:8765 2024-09-13T22:34:31.903 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:31 smithi105 ceph-mon[65554]: [13/Sep/2024:22:34:30] ENGINE Bus STARTED 2024-09-13T22:34:31.903 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:31 smithi105 ceph-mon[65554]: pgmap v4: 97 pgs: 97 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T22:34:31.903 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:31 smithi105 ceph-mon[65554]: mgrmap e41: smithi105.swmmea(active, since 2s), standbys: smithi118.rbhtgz 2024-09-13T22:34:31.903 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:31 smithi105 ceph-mon[65554]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:34:31.903 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:31 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/3370127994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:34:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:31 smithi118 ceph-mon[25717]: [13/Sep/2024:22:34:30] ENGINE Serving on https://172.21.15.105:7150 2024-09-13T22:34:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:31 smithi118 ceph-mon[25717]: [13/Sep/2024:22:34:30] ENGINE Error in HTTPServer.serve 2024-09-13T22:34:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Traceback (most recent call last): 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._connections.run(self.expiration_interval) 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._run(expiration_interval) 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s = self.context.wrap_socket( 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T22:34:32.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: return self.sslsocket_class._create( 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self.do_handshake() 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._sslobj.do_handshake() 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:31 smithi118 ceph-mon[25717]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:31 smithi118 ceph-mon[25717]: [13/Sep/2024:22:34:30] ENGINE Serving on http://172.21.15.105:8765 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:31 smithi118 ceph-mon[25717]: [13/Sep/2024:22:34:30] ENGINE Bus STARTED 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:31 smithi118 ceph-mon[25717]: pgmap v4: 97 pgs: 97 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:31 smithi118 ceph-mon[25717]: mgrmap e41: smithi105.swmmea(active, since 2s), standbys: smithi118.rbhtgz 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:31 smithi118 ceph-mon[25717]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:34:32.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:31 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/3370127994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:34:32.215 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:34:32.215 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:34:32.215 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:34:32.215 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:34:32.215 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:34:32.216 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:34:32.216 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:34:32.216 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "2/23 daemons upgraded", 2024-09-13T22:34:32.216 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "", 2024-09-13T22:34:32.216 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:34:32.216 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:34:32.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 119143 9.88 MB/sec execute 180 sec latency 52.420 ms 2024-09-13T22:34:32.767 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:34:32.911 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:32 smithi105 ceph-mon[65554]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:34:32.911 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:32 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2357804858' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:34:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:32 smithi118 ceph-mon[25717]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:34:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:32 smithi118 ceph-mon[25717]: from='client.? 172.21.15.105:0/2357804858' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:34:33.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 119552 9.90 MB/sec execute 181 sec latency 65.973 ms 2024-09-13T22:34:33.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:33 smithi105 ceph-mon[65554]: pgmap v5: 97 pgs: 97 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T22:34:33.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:33 smithi105 ceph-mon[65554]: mgrmap e42: smithi105.swmmea(active, since 4s), standbys: smithi118.rbhtgz 2024-09-13T22:34:33.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:33 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:33.937 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:33 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:33 smithi118 ceph-mon[25717]: pgmap v5: 97 pgs: 97 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T22:34:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:33 smithi118 ceph-mon[25717]: mgrmap e42: smithi105.swmmea(active, since 4s), standbys: smithi118.rbhtgz 2024-09-13T22:34:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:33 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:33 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:34.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 119971 9.94 MB/sec execute 182 sec latency 39.707 ms 2024-09-13T22:34:35.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 120143 9.93 MB/sec execute 183 sec latency 365.405 ms 2024-09-13T22:34:35.622 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:35 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:35.622 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:35 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:35.622 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:35 smithi105 ceph-mon[65554]: pgmap v6: 97 pgs: 97 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T22:34:35.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:35 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:35.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:35 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:35.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:35 smithi118 ceph-mon[25717]: pgmap v6: 97 pgs: 97 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T22:34:36.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 120498 9.93 MB/sec execute 184 sec latency 381.454 ms 2024-09-13T22:34:37.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 120786 9.89 MB/sec execute 185 sec latency 121.713 ms 2024-09-13T22:34:37.961 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:37 smithi105 ceph-mon[65554]: pgmap v7: 97 pgs: 97 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 4.8 MiB/s wr, 488 op/s 2024-09-13T22:34:38.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:37 smithi118 ceph-mon[25717]: pgmap v7: 97 pgs: 97 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 4.8 MiB/s wr, 488 op/s 2024-09-13T22:34:38.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 121066 9.86 MB/sec execute 186 sec latency 36.494 ms 2024-09-13T22:34:39.265 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:39 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:39.265 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:39 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:39.265 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:39 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:39.265 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:39 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:39.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:39 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:39.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:39 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:39.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:39 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:39.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:39 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:39.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 121351 9.85 MB/sec execute 187 sec latency 65.019 ms 2024-09-13T22:34:40.290 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:40 smithi105 ceph-mon[65554]: pgmap v8: 97 pgs: 97 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 3.7 MiB/s wr, 379 op/s 2024-09-13T22:34:40.290 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:40 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:40.290 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:40 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:40.290 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:40 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:40.290 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:40 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:34:40.291 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:40 smithi105 ceph-mon[65554]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:34:40.291 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:40 smithi105 ceph-mon[65554]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-13T22:34:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:40 smithi118 ceph-mon[25717]: pgmap v8: 97 pgs: 97 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 3.7 MiB/s wr, 379 op/s 2024-09-13T22:34:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:40 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:40 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:40 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:40 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:34:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:40 smithi118 ceph-mon[25717]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-13T22:34:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:40 smithi118 ceph-mon[25717]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-13T22:34:40.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 121684 9.85 MB/sec execute 188 sec latency 57.636 ms 2024-09-13T22:34:41.552 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:34:41.552 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:34:41.552 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:34:41.552 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:34:41.552 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:34:41.552 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:34:41.552 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.552 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.553 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.553 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.553 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.553 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.553 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:34:41.553 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:34:41.553 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:41 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:34:41.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 122067 9.84 MB/sec execute 189 sec latency 60.189 ms 2024-09-13T22:34:41.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:34:41.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T22:34:41.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:34:41.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-13T22:34:41.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: Updating smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:34:41.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: Updating smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T22:34:41.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:41.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:34:41.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:34:41.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:41 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T22:34:42.350 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:42 smithi105 ceph-mon[65554]: pgmap v9: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.1 MiB/s wr, 512 op/s 2024-09-13T22:34:42.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 122397 9.83 MB/sec execute 190 sec latency 57.211 ms 2024-09-13T22:34:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:42 smithi118 ceph-mon[25717]: pgmap v9: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.1 MiB/s wr, 512 op/s 2024-09-13T22:34:43.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 122736 9.82 MB/sec execute 191 sec latency 48.189 ms 2024-09-13T22:34:44.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:44 smithi105 ceph-mon[65554]: pgmap v10: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.7 MiB/s wr, 469 op/s 2024-09-13T22:34:44.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:44 smithi105 ceph-mon[65554]: Upgrade: Updating mon.smithi118 2024-09-13T22:34:44.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:44 smithi105 ceph-mon[65554]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:44.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:34:44.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:34:44.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:44.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:44 smithi105 ceph-mon[65554]: Deploying daemon mon.smithi118 on smithi118 2024-09-13T22:34:44.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:34:44.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 123038 9.81 MB/sec execute 192 sec latency 74.620 ms 2024-09-13T22:34:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:44 smithi118 ceph-mon[25717]: pgmap v10: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.7 MiB/s wr, 469 op/s 2024-09-13T22:34:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:44 smithi118 ceph-mon[25717]: Upgrade: Updating mon.smithi118 2024-09-13T22:34:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:44 smithi118 ceph-mon[25717]: from='mgr.34108 ' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:44 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:34:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:44 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:34:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:44 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:44 smithi118 ceph-mon[25717]: Deploying daemon mon.smithi118 on smithi118 2024-09-13T22:34:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:44 smithi118 ceph-mon[25717]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:34:45.191 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:44 smithi118 systemd[1]: Stopping Ceph mon.smithi118 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:34:45.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 123305 9.81 MB/sec execute 193 sec latency 77.343 ms 2024-09-13T22:34:45.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118[25701]: 2024-09-13T22:34:45.189+0000 7f3cb9e30700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi118 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:34:45.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118[25701]: 2024-09-13T22:34:45.189+0000 7f3cb9e30700 -1 mon.smithi118@1(peon) e2 *** Got Signal Terminated *** 2024-09-13T22:34:45.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:45 smithi118 podman[51411]: 2024-09-13 22:34:45.260839511 +0000 UTC m=+0.194223579 container died 17b16ac8afb2e81435a3ccd15b75552a8f5e1c5367f18162469ff95bd6280c01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, build-date=2022-05-03T08:36:31.336870, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_CLEAN=True, distribution-scope=public, name=centos-stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-13T22:34:45.959 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:45 smithi118 podman[51411]: 2024-09-13 22:34:45.673534207 +0000 UTC m=+0.606918318 container cleanup 17b16ac8afb2e81435a3ccd15b75552a8f5e1c5367f18162469ff95bd6280c01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, version=8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, distribution-scope=public, release=754, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, vcs-type=git, ceph=True, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T22:34:45.959 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:45 smithi118 bash[51411]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118 2024-09-13T22:34:46.317 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:45 smithi118 podman[51424]: 2024-09-13 22:34:45.958752232 +0000 UTC m=+0.696941570 container remove 17b16ac8afb2e81435a3ccd15b75552a8f5e1c5367f18162469ff95bd6280c01 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.display-name=CentOS Stream 8, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_CLEAN=True, architecture=x86_64, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, ceph=True) 2024-09-13T22:34:46.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 123657 9.84 MB/sec execute 194 sec latency 43.889 ms 2024-09-13T22:34:46.583 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:46 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi118.service: Deactivated successfully. 2024-09-13T22:34:46.583 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:46 smithi118 systemd[1]: Stopped Ceph mon.smithi118 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:34:46.583 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:46 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi118.service: Consumed 6.994s CPU time. 2024-09-13T22:34:46.844 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:46 smithi118 systemd[1]: Starting Ceph mon.smithi118 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:34:46.844 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:46 smithi118 podman[51548]: 2024-09-13 22:34:46.72028035 +0000 UTC m=+0.019735086 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:34:47.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:46 smithi118 podman[51548]: 2024-09-13 22:34:46.843778149 +0000 UTC m=+0.143232885 container create e2c0838729bcd956b282ca84a09ac0a04820a7f5ffb2d8d3d9311908606f1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=) 2024-09-13T22:34:47.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 123944 9.84 MB/sec execute 195 sec latency 57.718 ms 2024-09-13T22:34:47.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 podman[51548]: 2024-09-13 22:34:47.255862832 +0000 UTC m=+0.555317573 container init e2c0838729bcd956b282ca84a09ac0a04820a7f5ffb2d8d3d9311908606f1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 podman[51548]: 2024-09-13 22:34:47.260393479 +0000 UTC m=+0.559848215 container start e2c0838729bcd956b282ca84a09ac0a04820a7f5ffb2d8d3d9311908606f1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable), process ceph-mon, pid 2 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: pidfile_write: ignore empty --pid-file 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: load: jerasure load: lrc 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: RocksDB version: 7.9.2 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Git sha 0 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Compile date 2024-09-13 16:59:10 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: DB SUMMARY 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: DB Session ID: A7DX8JXR8KCLBK6XG40W 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: CURRENT file: CURRENT 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T22:34:47.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi118/store.db dir, Total Num: 1, files: 000024.sst 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi118/store.db: 000022.log size: 7580821 ; 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.error_if_exists: 0 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.create_if_missing: 0 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.paranoid_checks: 1 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.env: 0x55753ae56940 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.info_log: 0x55753cfeb8c0 2024-09-13T22:34:47.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.statistics: (nil) 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.use_fsync: 0 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_log_file_size: 0 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.allow_fallocate: 1 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.use_direct_reads: 0 2024-09-13T22:34:47.601 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.db_log_dir: 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.wal_dir: 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T22:34:47.602 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.write_buffer_manager: 0x55753cffadc0 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.unordered_write: 0 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T22:34:47.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.row_cache: None 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.wal_filter: None 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.two_write_queues: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.wal_compression: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.atomic_flush: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.log_readahead_size: 0 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T22:34:47.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_background_jobs: 2 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_background_compactions: -1 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_subcompactions: 1 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T22:34:47.605 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_open_files: -1 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_background_flushes: -1 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Compression algorithms supported: 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: kZSTD supported: 0 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: kXpressCompression supported: 0 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: kBZip2Compression supported: 0 2024-09-13T22:34:47.606 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: kLZ4Compression supported: 1 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: kZlibCompression supported: 1 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: kSnappyCompression supported: 1 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi118/store.db/MANIFEST-000009 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.merge_operator: 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_filter: None 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T22:34:47.607 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55753cfeb520) 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: cache_index_and_filter_blocks: 1 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: pin_top_level_index_and_filter: 1 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: index_type: 0 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: data_block_index_type: 0 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: index_shortening: 1 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: checksum: 4 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: no_block_cache: 0 2024-09-13T22:34:47.608 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache: 0x55753cfdf1f0 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache_name: BinnedLRUCache 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache_options: 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: capacity : 536870912 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: num_shard_bits : 4 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: strict_capacity_limit : 0 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: high_pri_pool_ratio: 0.000 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache_compressed: (nil) 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: persistent_cache: (nil) 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_size: 4096 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_size_deviation: 10 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_restart_interval: 16 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: index_block_restart_interval: 1 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: metadata_block_size: 4096 2024-09-13T22:34:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: partition_filters: 0 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: use_delta_encoding: 1 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: filter_policy: bloomfilter 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: whole_key_filtering: 1 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: verify_compression: 0 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: read_amp_bytes_per_bit: 0 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: format_version: 5 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: enable_index_compression: 1 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_align: 0 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: max_auto_readahead_size: 262144 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: prepopulate_block_cache: 0 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: initial_auto_readahead_size: 8192 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression: NoCompression 2024-09-13T22:34:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.num_levels: 7 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T22:34:47.611 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T22:34:47.612 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T22:34:47.613 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.inplace_update_support: 0 2024-09-13T22:34:47.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.bloom_locality: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.max_successive_merges: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.ttl: 2592000 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.enable_blob_files: false 2024-09-13T22:34:47.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.min_blob_size: 0 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi118/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12232, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-13T22:34:47.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 70d48f60-5ce1-4eae-9317-ee673369d9bb 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266887293702, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266887311745, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4644364, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12237, "largest_seqno": 13928, "table_properties": {"data_size": 4636677, "index_size": 4914, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 15656, "raw_average_key_size": 23, "raw_value_size": 4621923, "raw_average_value_size": 6837, "num_data_blocks": 223, "num_entries": 676, "num_filter_entries": 676, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726266887, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "70d48f60-5ce1-4eae-9317-ee673369d9bb", "db_session_id": "A7DX8JXR8KCLBK6XG40W", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726266887311891, "job": 1, "event": "recovery_finished"} 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi118/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55753d00ce00 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: DB pointer 0x55753d108000 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** DB Stats ** 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:34:47.617 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** Compaction Stats [default] ** 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: L0 1/0 4.43 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 555.0 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: L6 1/0 6.84 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Sum 2/0 11.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 555.0 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 555.0 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:34:47.618 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** Compaction Stats [default] ** 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 555.0 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative compaction: 0.00 GB write, 207.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval compaction: 0.00 GB write, 207.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T22:34:47.619 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Block cache BinnedLRUCache@0x55753cfdf1f0#2 capacity: 512.00 MB usage: 6.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: starting mon.smithi118 rank 1 at public addrs [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] at bind addrs [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi118 fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: mon.smithi118@-1(???) e2 preinit fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: mon.smithi118@-1(???).mds e7 new map 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: mon.smithi118@-1(???).mds e7 print_map 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: e7 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: legacy client fscid: 1 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Filesystem 'foofs' (1) 2024-09-13T22:34:47.620 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: fs_name foofs 2024-09-13T22:34:47.621 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: epoch 5 2024-09-13T22:34:47.621 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T22:34:47.621 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: created 2024-09-13T22:28:57.698196+0000 2024-09-13T22:34:47.621 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: modified 2024-09-13T22:29:02.728668+0000 2024-09-13T22:34:47.621 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: tableserver 0 2024-09-13T22:34:47.624 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: root 0 2024-09-13T22:34:47.624 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: session_timeout 60 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: session_autoclose 300 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: max_file_size 1099511627776 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: max_xattr_size 65536 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: required_client_features {} 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: last_failure 0 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: last_failure_osd_epoch 0 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: max_mds 1 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: in 0 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: up {0=14486} 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: failed 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: damaged 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: stopped 2024-09-13T22:34:47.625 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: data_pools [3] 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: metadata_pool 2 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: inline_data disabled 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: balancer 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: bal_rank_mask -1 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: standby_count_wanted 1 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: qdb_cluster leader: 0 members: 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: [mds.foofs.smithi105.ddoxtb{0:14486} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.105:6834/1075044016,v1:172.21.15.105:6835/1075044016] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Standby daemons: 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: [mds.foofs.smithi118.izbofj{-1:24281} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: mon.smithi118@-1(???).osd e74 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: mon.smithi118@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: mon.smithi118@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:34:47.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: mon.smithi118@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T22:34:47.627 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 ceph-mon[51566]: mon.smithi118@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-13T22:34:47.627 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 bash[51548]: e2c0838729bcd956b282ca84a09ac0a04820a7f5ffb2d8d3d9311908606f1b43 2024-09-13T22:34:47.627 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:47 smithi118 systemd[1]: Started Ceph mon.smithi118 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:34:48.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 124302 9.82 MB/sec execute 196 sec latency 47.947 ms 2024-09-13T22:34:48.815 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:48 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:48.815 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:48 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:49.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:48 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:49.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:48 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:49.552 INFO:teuthology.orchestra.run.smithi105.stdout: 5 124562 9.79 MB/sec execute 197 sec latency 74.733 ms 2024-09-13T22:34:49.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:49.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:49.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: mon.smithi105 calling monitor election 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: mon.smithi118 calling monitor election 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: mon.smithi105 is new leader, mons smithi105,smithi118 in quorum (ranks 0,1) 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: monmap epoch 3 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: last_changed 2024-09-13T22:34:48.651881+0000 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: created 2024-09-13T22:21:27.083377+0000 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: min_mon_release 19 (squid) 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: election_strategy: 1 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: 0: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.smithi105 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: 1: [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon.smithi118 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: mgrmap e42: smithi105.swmmea(active, since 19s), standbys: smithi118.rbhtgz 2024-09-13T22:34:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: overall HEALTH_OK 2024-09-13T22:34:49.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:49.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:49.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:34:49.870 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:49 smithi105 ceph-mon[65554]: pgmap v13: 97 pgs: 97 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 386 op/s 2024-09-13T22:34:50.083 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi105"}]: dispatch 2024-09-13T22:34:50.084 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-13T22:34:50.084 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: mon.smithi105 calling monitor election 2024-09-13T22:34:50.084 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: mon.smithi118 calling monitor election 2024-09-13T22:34:50.084 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: mon.smithi105 is new leader, mons smithi105,smithi118 in quorum (ranks 0,1) 2024-09-13T22:34:50.084 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: monmap epoch 3 2024-09-13T22:34:50.084 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: fsid 782bdfc8-721e-11ef-bceb-c7b262605968 2024-09-13T22:34:50.084 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: last_changed 2024-09-13T22:34:48.651881+0000 2024-09-13T22:34:50.084 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: created 2024-09-13T22:21:27.083377+0000 2024-09-13T22:34:50.084 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: min_mon_release 19 (squid) 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: election_strategy: 1 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: 0: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.smithi105 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: 1: [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon.smithi118 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: mgrmap e42: smithi105.swmmea(active, since 19s), standbys: smithi118.rbhtgz 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: overall HEALTH_OK 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:34:50.085 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:49 smithi118 ceph-mon[51566]: pgmap v13: 97 pgs: 97 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 386 op/s 2024-09-13T22:34:50.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 124886 9.78 MB/sec execute 198 sec latency 64.364 ms 2024-09-13T22:34:51.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 125284 9.79 MB/sec execute 199 sec latency 40.220 ms 2024-09-13T22:34:51.938 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:51 smithi105 ceph-mon[65554]: pgmap v14: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 568 op/s 2024-09-13T22:34:52.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:51 smithi118 ceph-mon[51566]: pgmap v14: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 568 op/s 2024-09-13T22:34:52.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 125709 9.79 MB/sec execute 200 sec latency 75.046 ms 2024-09-13T22:34:53.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 125984 9.77 MB/sec execute 201 sec latency 61.651 ms 2024-09-13T22:34:53.786 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:53.786 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:53.786 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:53 smithi105 ceph-mon[65554]: pgmap v15: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 383 op/s 2024-09-13T22:34:53.786 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:53.786 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:53 smithi118 ceph-mon[51566]: pgmap v15: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 383 op/s 2024-09-13T22:34:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:54.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 126346 9.77 MB/sec execute 202 sec latency 103.144 ms 2024-09-13T22:34:55.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 126622 9.77 MB/sec execute 203 sec latency 49.790 ms 2024-09-13T22:34:56.093 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:55 smithi105 ceph-mon[65554]: pgmap v16: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 383 op/s 2024-09-13T22:34:56.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:55 smithi118 ceph-mon[51566]: pgmap v16: 97 pgs: 97 active+clean; 353 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 383 op/s 2024-09-13T22:34:56.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 126884 9.76 MB/sec execute 204 sec latency 98.894 ms 2024-09-13T22:34:57.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 127177 9.77 MB/sec execute 205 sec latency 68.987 ms 2024-09-13T22:34:58.136 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: pgmap v17: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 601 op/s 2024-09-13T22:34:58.136 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:58.136 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:58.136 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:58.136 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:34:58.136 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:58.137 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:58.137 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: Reconfiguring mon.smithi105 (monmap changed)... 2024-09-13T22:34:58.137 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:34:58.137 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:34:58.137 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:58.137 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:57 smithi105 ceph-mon[65554]: Reconfiguring daemon mon.smithi105 on smithi105 2024-09-13T22:34:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: pgmap v17: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 601 op/s 2024-09-13T22:34:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:34:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:34:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: Reconfiguring mon.smithi105 (monmap changed)... 2024-09-13T22:34:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:34:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:34:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:34:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:57 smithi118 ceph-mon[51566]: Reconfiguring daemon mon.smithi105 on smithi105 2024-09-13T22:34:58.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 127387 9.77 MB/sec execute 206 sec latency 95.777 ms 2024-09-13T22:34:59.170 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:34:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:34:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:34:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:34:59.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 127610 9.75 MB/sec execute 207 sec latency 72.943 ms 2024-09-13T22:35:00.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:00 smithi105 ceph-mon[65554]: pgmap v18: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-13T22:35:00.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:00.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:00.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:00 smithi105 ceph-mon[65554]: Reconfiguring mgr.smithi105.swmmea (monmap changed)... 2024-09-13T22:35:00.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi105.swmmea", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:35:00.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:35:00.430 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:00.430 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:00 smithi105 ceph-mon[65554]: Reconfiguring daemon mgr.smithi105.swmmea on smithi105 2024-09-13T22:35:00.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 127920 9.72 MB/sec execute 208 sec latency 35.361 ms 2024-09-13T22:35:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:00 smithi118 ceph-mon[51566]: pgmap v18: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-13T22:35:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:00 smithi118 ceph-mon[51566]: Reconfiguring mgr.smithi105.swmmea (monmap changed)... 2024-09-13T22:35:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi105.swmmea", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:35:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:35:00.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:00.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:00 smithi118 ceph-mon[51566]: Reconfiguring daemon mgr.smithi105.swmmea on smithi105 2024-09-13T22:35:01.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 128130 9.69 MB/sec execute 209 sec latency 99.734 ms 2024-09-13T22:35:02.240 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:02 smithi105 ceph-mon[65554]: pgmap v19: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 554 op/s 2024-09-13T22:35:02.240 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:02 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:02.240 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:02 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:02.240 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:02 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi105", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:35:02.241 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:02 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:02.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 128531 9.69 MB/sec execute 210 sec latency 46.184 ms 2024-09-13T22:35:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:02 smithi118 ceph-mon[51566]: pgmap v19: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 554 op/s 2024-09-13T22:35:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:02 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:02 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:02 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi105", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:35:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:02 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:03.250 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:03 smithi105 ceph-mon[65554]: Reconfiguring crash.smithi105 (monmap changed)... 2024-09-13T22:35:03.250 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:03 smithi105 ceph-mon[65554]: Reconfiguring daemon crash.smithi105 on smithi105 2024-09-13T22:35:03.355 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:35:03.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 128812 9.68 MB/sec execute 211 sec latency 69.109 ms 2024-09-13T22:35:03.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:03 smithi118 ceph-mon[51566]: Reconfiguring crash.smithi105 (monmap changed)... 2024-09-13T22:35:03.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:03 smithi118 ceph-mon[51566]: Reconfiguring daemon crash.smithi105 on smithi105 2024-09-13T22:35:04.275 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:04 smithi105 ceph-mon[65554]: pgmap v20: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 372 op/s 2024-09-13T22:35:04.275 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:04 smithi105 ceph-mon[65554]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:04.275 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:04 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:04.275 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:04 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:04.275 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:04 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:35:04.275 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:04 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:04.354 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:35:04.354 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (2m) 26s ago 12m 18.8M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:35:04.354 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (12m) 26s ago 12m 6405k - 17.2.0 e1d6a67b021e eb5cdb0f830f 2024-09-13T22:35:04.354 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (11m) 11s ago 11m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:35:04.354 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (2m) 26s ago 11m 47.2M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:35:04.354 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (6m) 26s ago 6m 37.6M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (6m) 11s ago 6m 14.1M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (2m) 26s ago 13m 476M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (69s) 11s ago 11m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (50s) 26s ago 13m 32.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (17s) 11s ago 11m 25.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (5m) 26s ago 5m 260M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (5m) 11s ago 5m 81.0M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (2m) 26s ago 11m 8786k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (2m) 11s ago 11m 15.1M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (10m) 26s ago 10m 465M 1936M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (10m) 26s ago 10m 388M 1936M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (9m) 26s ago 9m 276M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:35:04.355 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (9m) 26s ago 9m 418M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:35:04.356 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (8m) 11s ago 8m 449M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:35:04.356 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (8m) 11s ago 8m 479M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:35:04.356 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (7m) 11s ago 7m 407M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:35:04.356 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (7m) 11s ago 7m 579M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:35:04.356 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (75s) 26s ago 11m 53.5M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:35:04.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 129177 9.68 MB/sec execute 212 sec latency 67.166 ms 2024-09-13T22:35:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:04 smithi118 ceph-mon[51566]: pgmap v20: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 372 op/s 2024-09-13T22:35:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:04 smithi118 ceph-mon[51566]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:04 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:04 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:04 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:35:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:04 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:35:04.867 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:35:04.868 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:35:04.868 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:35:04.868 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:35:04.868 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:35:04.868 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T22:35:04.868 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T22:35:04.868 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:35:04.868 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:35:05.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:05 smithi105 ceph-mon[65554]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T22:35:05.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:05 smithi105 ceph-mon[65554]: Reconfiguring daemon osd.0 on smithi105 2024-09-13T22:35:05.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:05 smithi105 ceph-mon[65554]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:05.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:05 smithi105 ceph-mon[65554]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:05.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:05 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/1638121448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:05.295 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:35:05.295 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:35:05.295 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:35:05.295 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:35:05.295 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:35:05.295 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:35:05.295 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:35:05.295 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:35:05.295 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "4/23 daemons upgraded", 2024-09-13T22:35:05.296 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T22:35:05.296 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:35:05.296 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:35:05.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 129527 9.67 MB/sec execute 213 sec latency 57.443 ms 2024-09-13T22:35:05.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:05 smithi118 ceph-mon[51566]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T22:35:05.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:05 smithi118 ceph-mon[51566]: Reconfiguring daemon osd.0 on smithi105 2024-09-13T22:35:05.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:05 smithi118 ceph-mon[51566]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:05.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:05 smithi118 ceph-mon[51566]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:05.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:05 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/1638121448' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:05.778 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:35:06.293 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:06 smithi105 ceph-mon[65554]: pgmap v21: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 372 op/s 2024-09-13T22:35:06.293 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:06 smithi105 ceph-mon[65554]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:06.293 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:06 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2598802242' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:35:06.293 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:06 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:06.293 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:06 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:06.294 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:06 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T22:35:06.294 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:06 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:06.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 129792 9.64 MB/sec execute 214 sec latency 42.667 ms 2024-09-13T22:35:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:06 smithi118 ceph-mon[51566]: pgmap v21: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 372 op/s 2024-09-13T22:35:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:06 smithi118 ceph-mon[51566]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:06 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2598802242' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:35:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:06 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:06 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:06 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T22:35:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:06 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:07.312 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:07 smithi105 ceph-mon[65554]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T22:35:07.312 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:07 smithi105 ceph-mon[65554]: Reconfiguring daemon osd.1 on smithi105 2024-09-13T22:35:07.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 130083 9.65 MB/sec execute 215 sec latency 55.413 ms 2024-09-13T22:35:07.596 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:07 smithi118 ceph-mon[51566]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T22:35:07.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:07 smithi118 ceph-mon[51566]: Reconfiguring daemon osd.1 on smithi105 2024-09-13T22:35:08.321 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:08 smithi105 ceph-mon[65554]: pgmap v22: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-09-13T22:35:08.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 130466 9.67 MB/sec execute 216 sec latency 38.491 ms 2024-09-13T22:35:08.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:08 smithi118 ceph-mon[51566]: pgmap v22: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-09-13T22:35:09.376 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:09.376 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:09.376 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:09 smithi105 ceph-mon[65554]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T22:35:09.376 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T22:35:09.376 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:09.377 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:09 smithi105 ceph-mon[65554]: Reconfiguring daemon osd.2 on smithi105 2024-09-13T22:35:09.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 130865 9.71 MB/sec execute 217 sec latency 72.449 ms 2024-09-13T22:35:09.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:09.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:09.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:09 smithi118 ceph-mon[51566]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T22:35:09.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T22:35:09.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:09.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:09 smithi118 ceph-mon[51566]: Reconfiguring daemon osd.2 on smithi105 2024-09-13T22:35:10.385 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:10 smithi105 ceph-mon[65554]: pgmap v23: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.5 MiB/s wr, 358 op/s 2024-09-13T22:35:10.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 131217 9.70 MB/sec execute 218 sec latency 46.759 ms 2024-09-13T22:35:10.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:10 smithi118 ceph-mon[51566]: pgmap v23: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.5 MiB/s wr, 358 op/s 2024-09-13T22:35:11.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 131514 9.67 MB/sec execute 219 sec latency 59.699 ms 2024-09-13T22:35:11.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:11 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:11.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:11 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:11.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:11 smithi105 ceph-mon[65554]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T22:35:11.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:11 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T22:35:11.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:11 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:11.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:11 smithi105 ceph-mon[65554]: Reconfiguring daemon osd.3 on smithi105 2024-09-13T22:35:11.652 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:11 smithi105 ceph-mon[65554]: pgmap v24: 97 pgs: 97 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 553 op/s 2024-09-13T22:35:11.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:11 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:11.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:11 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:11.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:11 smithi118 ceph-mon[51566]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T22:35:11.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:11 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T22:35:11.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:11 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:11.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:11 smithi118 ceph-mon[51566]: Reconfiguring daemon osd.3 on smithi105 2024-09-13T22:35:11.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:11 smithi118 ceph-mon[51566]: pgmap v24: 97 pgs: 97 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 553 op/s 2024-09-13T22:35:12.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 131748 9.64 MB/sec execute 220 sec latency 76.622 ms 2024-09-13T22:35:13.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 131967 9.63 MB/sec execute 221 sec latency 88.944 ms 2024-09-13T22:35:13.745 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:13.745 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:13.745 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:13 smithi105 ceph-mon[65554]: Reconfiguring mds.foofs.smithi105.ddoxtb (monmap changed)... 2024-09-13T22:35:13.745 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi105.ddoxtb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:35:13.745 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:13.745 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:13 smithi105 ceph-mon[65554]: Reconfiguring daemon mds.foofs.smithi105.ddoxtb on smithi105 2024-09-13T22:35:13.745 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:13 smithi105 ceph-mon[65554]: pgmap v25: 97 pgs: 97 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 398 op/s 2024-09-13T22:35:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:13 smithi118 ceph-mon[51566]: Reconfiguring mds.foofs.smithi105.ddoxtb (monmap changed)... 2024-09-13T22:35:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi105.ddoxtb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:35:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:13 smithi118 ceph-mon[51566]: Reconfiguring daemon mds.foofs.smithi105.ddoxtb on smithi105 2024-09-13T22:35:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:13 smithi118 ceph-mon[51566]: pgmap v25: 97 pgs: 97 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 398 op/s 2024-09-13T22:35:14.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 132262 9.63 MB/sec execute 222 sec latency 77.917 ms 2024-09-13T22:35:14.784 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:35:14.784 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:14.784 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:14.784 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:35:14.784 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:15.054 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:35:15.054 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:15.054 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:15.054 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:35:15.055 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:15.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 132667 9.63 MB/sec execute 223 sec latency 41.242 ms 2024-09-13T22:35:15.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:15 smithi105 ceph-mon[65554]: Reconfiguring crash.smithi118 (monmap changed)... 2024-09-13T22:35:15.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:15 smithi105 ceph-mon[65554]: Reconfiguring daemon crash.smithi118 on smithi118 2024-09-13T22:35:15.811 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:15 smithi105 ceph-mon[65554]: pgmap v26: 97 pgs: 97 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-13T22:35:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:15 smithi118 ceph-mon[51566]: Reconfiguring crash.smithi118 (monmap changed)... 2024-09-13T22:35:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:15 smithi118 ceph-mon[51566]: Reconfiguring daemon crash.smithi118 on smithi118 2024-09-13T22:35:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:15 smithi118 ceph-mon[51566]: pgmap v26: 97 pgs: 97 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-13T22:35:16.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 133013 9.62 MB/sec execute 224 sec latency 62.385 ms 2024-09-13T22:35:17.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:17 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:17.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:17 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:17.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:17 smithi105 ceph-mon[65554]: Reconfiguring mgr.smithi118.rbhtgz (monmap changed)... 2024-09-13T22:35:17.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:17 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:35:17.334 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:17 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:35:17.335 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:17 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:17.335 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:17 smithi105 ceph-mon[65554]: Reconfiguring daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:35:17.478 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:17 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:17.478 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:17 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:17.478 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:17 smithi118 ceph-mon[51566]: Reconfiguring mgr.smithi118.rbhtgz (monmap changed)... 2024-09-13T22:35:17.478 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:17 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rbhtgz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T22:35:17.478 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:17 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T22:35:17.478 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:17 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:17.478 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:17 smithi118 ceph-mon[51566]: Reconfiguring daemon mgr.smithi118.rbhtgz on smithi118 2024-09-13T22:35:17.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 133304 9.59 MB/sec execute 225 sec latency 91.804 ms 2024-09-13T22:35:18.166 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:18 smithi105 ceph-mon[65554]: pgmap v27: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.8 MiB/s wr, 586 op/s 2024-09-13T22:35:18.166 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:18 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:18.166 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:18 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:18.166 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:18 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:35:18.166 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:18 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:35:18.166 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:18 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:18.472 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:18 smithi118 ceph-mon[51566]: pgmap v27: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.8 MiB/s wr, 586 op/s 2024-09-13T22:35:18.472 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:18 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:18.472 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:18 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:18.472 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:18 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T22:35:18.472 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:18 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T22:35:18.472 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:18 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:18.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 133823 9.63 MB/sec execute 226 sec latency 47.809 ms 2024-09-13T22:35:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:19 smithi118 ceph-mon[51566]: Reconfiguring mon.smithi118 (monmap changed)... 2024-09-13T22:35:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:19 smithi118 ceph-mon[51566]: Reconfiguring daemon mon.smithi118 on smithi118 2024-09-13T22:35:19.367 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:19 smithi105 ceph-mon[65554]: Reconfiguring mon.smithi118 (monmap changed)... 2024-09-13T22:35:19.367 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:19 smithi105 ceph-mon[65554]: Reconfiguring daemon mon.smithi118 on smithi118 2024-09-13T22:35:19.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 134254 9.67 MB/sec execute 227 sec latency 41.539 ms 2024-09-13T22:35:20.342 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:20 smithi118 ceph-mon[51566]: pgmap v28: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 382 op/s 2024-09-13T22:35:20.343 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:20.343 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:20.343 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T22:35:20.343 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:20.384 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:20 smithi105 ceph-mon[65554]: pgmap v28: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 382 op/s 2024-09-13T22:35:20.384 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:20.384 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:20.384 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T22:35:20.384 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:20.553 INFO:teuthology.orchestra.run.smithi105.stdout: 5 134592 9.67 MB/sec execute 228 sec latency 51.499 ms 2024-09-13T22:35:21.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:21 smithi118 ceph-mon[51566]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T22:35:21.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:21 smithi118 ceph-mon[51566]: Reconfiguring daemon osd.4 on smithi118 2024-09-13T22:35:21.395 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:21 smithi105 ceph-mon[65554]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T22:35:21.395 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:21 smithi105 ceph-mon[65554]: Reconfiguring daemon osd.4 on smithi118 2024-09-13T22:35:21.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 134931 9.66 MB/sec execute 229 sec latency 111.713 ms 2024-09-13T22:35:22.435 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:22 smithi105 ceph-mon[65554]: pgmap v29: 97 pgs: 97 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 609 op/s 2024-09-13T22:35:22.435 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:22 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:22.435 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:22 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:22.435 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:22 smithi105 ceph-mon[65554]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T22:35:22.435 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:22 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T22:35:22.435 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:22 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:22.435 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:22 smithi105 ceph-mon[65554]: Reconfiguring daemon osd.5 on smithi118 2024-09-13T22:35:22.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 135167 9.63 MB/sec execute 230 sec latency 74.392 ms 2024-09-13T22:35:22.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:22 smithi118 ceph-mon[51566]: pgmap v29: 97 pgs: 97 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 609 op/s 2024-09-13T22:35:22.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:22 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:22.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:22 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:22.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:22 smithi118 ceph-mon[51566]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T22:35:22.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:22 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T22:35:22.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:22 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:22.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:22 smithi118 ceph-mon[51566]: Reconfiguring daemon osd.5 on smithi118 2024-09-13T22:35:23.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 135526 9.63 MB/sec execute 231 sec latency 60.995 ms 2024-09-13T22:35:23.983 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:23 smithi105 ceph-mon[65554]: pgmap v30: 97 pgs: 97 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-09-13T22:35:23.983 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:23 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:23.983 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:23 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:23.983 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:23 smithi105 ceph-mon[65554]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T22:35:23.983 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:23 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T22:35:23.983 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:23 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:23.983 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:23 smithi105 ceph-mon[65554]: Reconfiguring daemon osd.6 on smithi118 2024-09-13T22:35:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:23 smithi118 ceph-mon[51566]: pgmap v30: 97 pgs: 97 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-09-13T22:35:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:23 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:23 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:23 smithi118 ceph-mon[51566]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T22:35:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:23 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T22:35:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:23 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:23 smithi118 ceph-mon[51566]: Reconfiguring daemon osd.6 on smithi118 2024-09-13T22:35:24.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 135946 9.63 MB/sec execute 232 sec latency 90.330 ms 2024-09-13T22:35:25.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 136380 9.64 MB/sec execute 233 sec latency 50.324 ms 2024-09-13T22:35:25.806 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:25.806 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:25.806 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:25 smithi105 ceph-mon[65554]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T22:35:25.806 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T22:35:25.806 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:25.807 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:25 smithi105 ceph-mon[65554]: Reconfiguring daemon osd.7 on smithi118 2024-09-13T22:35:25.807 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:25 smithi105 ceph-mon[65554]: pgmap v31: 97 pgs: 97 active+clean; 358 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.7 MiB/s wr, 499 op/s 2024-09-13T22:35:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:25 smithi118 ceph-mon[51566]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T22:35:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T22:35:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:25 smithi118 ceph-mon[51566]: Reconfiguring daemon osd.7 on smithi118 2024-09-13T22:35:25.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:25 smithi118 ceph-mon[51566]: pgmap v31: 97 pgs: 97 active+clean; 358 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.7 MiB/s wr, 499 op/s 2024-09-13T22:35:26.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 136775 9.63 MB/sec execute 234 sec latency 55.899 ms 2024-09-13T22:35:27.221 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:27.221 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:27.221 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:27 smithi118 ceph-mon[51566]: Reconfiguring mds.foofs.smithi118.izbofj (monmap changed)... 2024-09-13T22:35:27.221 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.izbofj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:35:27.221 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:27.221 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:27 smithi118 ceph-mon[51566]: Reconfiguring daemon mds.foofs.smithi118.izbofj on smithi118 2024-09-13T22:35:27.320 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:27.320 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:27.320 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:27 smithi105 ceph-mon[65554]: Reconfiguring mds.foofs.smithi118.izbofj (monmap changed)... 2024-09-13T22:35:27.320 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.izbofj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:35:27.320 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:27.320 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:27 smithi105 ceph-mon[65554]: Reconfiguring daemon mds.foofs.smithi118.izbofj on smithi118 2024-09-13T22:35:27.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 137152 9.64 MB/sec execute 235 sec latency 50.294 ms 2024-09-13T22:35:28.339 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: pgmap v32: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 625 op/s 2024-09-13T22:35:28.339 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:28.339 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:28.339 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:35:28.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:28.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:28.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:28.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi105"}]: dispatch 2024-09-13T22:35:28.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi105"}]': finished 2024-09-13T22:35:28.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi118"}]: dispatch 2024-09-13T22:35:28.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi118"}]': finished 2024-09-13T22:35:28.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 137481 9.65 MB/sec execute 236 sec latency 72.749 ms 2024-09-13T22:35:28.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: pgmap v32: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 625 op/s 2024-09-13T22:35:28.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:28.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:28.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:35:28.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:28.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:28.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:28.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi105"}]: dispatch 2024-09-13T22:35:28.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi105"}]': finished 2024-09-13T22:35:28.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi118"}]: dispatch 2024-09-13T22:35:28.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi118"}]': finished 2024-09-13T22:35:29.355 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:29 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all mon 2024-09-13T22:35:29.355 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:35:29.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 137816 9.67 MB/sec execute 237 sec latency 58.214 ms 2024-09-13T22:35:29.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:29 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all mon 2024-09-13T22:35:29.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:35:30.377 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:30 smithi105 ceph-mon[65554]: pgmap v33: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-13T22:35:30.377 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:30.377 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi105", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:35:30.377 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:30.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 138124 9.67 MB/sec execute 238 sec latency 62.598 ms 2024-09-13T22:35:30.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:30 smithi118 ceph-mon[51566]: pgmap v33: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-13T22:35:30.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:30.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi105", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:35:30.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:31 smithi118 ceph-mon[51566]: Upgrade: Updating crash.smithi105 (1/2) 2024-09-13T22:35:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:31 smithi118 ceph-mon[51566]: Deploying daemon crash.smithi105 on smithi105 2024-09-13T22:35:31.386 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:31 smithi105 ceph-mon[65554]: Upgrade: Updating crash.smithi105 (1/2) 2024-09-13T22:35:31.386 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:31 smithi105 ceph-mon[65554]: Deploying daemon crash.smithi105 on smithi105 2024-09-13T22:35:31.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 138505 9.65 MB/sec execute 239 sec latency 74.164 ms 2024-09-13T22:35:32.411 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:32 smithi105 ceph-mon[65554]: pgmap v34: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.9 MiB/s wr, 662 op/s 2024-09-13T22:35:32.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 138685 9.63 MB/sec execute 240 sec latency 58.294 ms 2024-09-13T22:35:32.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:32 smithi118 ceph-mon[51566]: pgmap v34: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.9 MiB/s wr, 662 op/s 2024-09-13T22:35:33.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 138978 9.62 MB/sec execute 241 sec latency 72.645 ms 2024-09-13T22:35:34.211 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:34 smithi105 ceph-mon[65554]: pgmap v35: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 436 op/s 2024-09-13T22:35:34.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 139384 9.62 MB/sec execute 242 sec latency 51.331 ms 2024-09-13T22:35:34.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:34 smithi118 ceph-mon[51566]: pgmap v35: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 436 op/s 2024-09-13T22:35:35.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 139872 9.63 MB/sec execute 243 sec latency 58.890 ms 2024-09-13T22:35:36.231 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:36 smithi105 ceph-mon[65554]: pgmap v36: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.9 MiB/s wr, 526 op/s 2024-09-13T22:35:36.231 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:36 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:36.231 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:36 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:36.381 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:35:36.525 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:36 smithi118 ceph-mon[51566]: pgmap v36: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.9 MiB/s wr, 526 op/s 2024-09-13T22:35:36.525 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:36 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:36.526 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:36 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:36.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 140360 9.64 MB/sec execute 244 sec latency 30.932 ms 2024-09-13T22:35:37.245 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:37 smithi105 ceph-mon[65554]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:37.486 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:35:37.486 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (3m) 59s ago 12m 18.8M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 starting - - - - 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (12m) 44s ago 12m 6413k - 17.2.0 e1d6a67b021e 3ef9492f409d 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (3m) 59s ago 12m 47.2M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (6m) 59s ago 6m 37.6M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (6m) 44s ago 6m 14.1M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (2m) 59s ago 13m 476M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (102s) 44s ago 11m 438M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (84s) 59s ago 14m 32.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (50s) 44s ago 11m 25.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (6m) 59s ago 6m 260M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (6m) 44s ago 6m 81.0M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:35:37.487 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (3m) 59s ago 12m 8786k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (2m) 44s ago 11m 15.1M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (11m) 59s ago 11m 465M 1936M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (10m) 59s ago 10m 388M 1936M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (10m) 59s ago 10m 276M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (9m) 59s ago 9m 418M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (9m) 44s ago 9m 449M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (8m) 44s ago 8m 479M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (8m) 44s ago 8m 407M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (8m) 44s ago 8m 579M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:35:37.488 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (108s) 59s ago 11m 53.5M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:35:37.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 141007 9.68 MB/sec execute 245 sec latency 22.880 ms 2024-09-13T22:35:37.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:37 smithi118 ceph-mon[51566]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:38.209 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:35:38.209 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:35:38.209 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:35:38.210 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:35:38.210 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:35:38.210 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:35:38.210 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:35:38.210 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:35:38.210 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T22:35:38.211 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:35:38.211 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:35:38.211 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:35:38.211 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:35:38.211 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:35:38.211 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T22:35:38.212 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T22:35:38.212 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:35:38.212 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:35:38.504 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:38 smithi105 ceph-mon[65554]: pgmap v37: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.7 MiB/s wr, 550 op/s 2024-09-13T22:35:38.504 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:38 smithi105 ceph-mon[65554]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:38.504 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:38 smithi105 ceph-mon[65554]: Upgrade: Updating crash.smithi118 (2/2) 2024-09-13T22:35:38.504 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:38 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:38.504 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:38 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:35:38.504 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:38 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:38.505 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:38 smithi105 ceph-mon[65554]: Deploying daemon crash.smithi118 on smithi118 2024-09-13T22:35:38.505 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:38 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/3767265864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:38.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 141467 9.72 MB/sec execute 246 sec latency 25.851 ms 2024-09-13T22:35:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:38 smithi118 ceph-mon[51566]: pgmap v37: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.7 MiB/s wr, 550 op/s 2024-09-13T22:35:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:38 smithi118 ceph-mon[51566]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:38 smithi118 ceph-mon[51566]: Upgrade: Updating crash.smithi118 (2/2) 2024-09-13T22:35:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:38 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:38 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T22:35:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:38 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:38.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:38 smithi118 ceph-mon[51566]: Deploying daemon crash.smithi118 on smithi118 2024-09-13T22:35:38.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:38 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/3767265864' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "4/23 daemons upgraded", 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading crash daemons", 2024-09-13T22:35:38.696 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:35:38.697 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:35:39.284 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:35:39.514 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:39 smithi105 ceph-mon[65554]: from='client.44119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:39.514 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:39 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/837367605' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:35:39.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 141830 9.71 MB/sec execute 247 sec latency 45.883 ms 2024-09-13T22:35:39.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:39 smithi118 ceph-mon[51566]: from='client.44119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:39.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:39 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/837367605' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:35:40.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:40 smithi105 ceph-mon[65554]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:40.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:40 smithi105 ceph-mon[65554]: pgmap v38: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 418 op/s 2024-09-13T22:35:40.542 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:40 smithi118 ceph-mon[51566]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:35:40.542 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:40 smithi118 ceph-mon[51566]: pgmap v38: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 418 op/s 2024-09-13T22:35:40.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 142183 9.70 MB/sec execute 248 sec latency 49.048 ms 2024-09-13T22:35:41.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 142674 9.72 MB/sec execute 249 sec latency 32.601 ms 2024-09-13T22:35:42.546 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:42 smithi105 ceph-mon[65554]: pgmap v39: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.3 MiB/s wr, 707 op/s 2024-09-13T22:35:42.546 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:42 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:42.546 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:42 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:42.546 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:42 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:35:42.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 143168 9.72 MB/sec execute 250 sec latency 85.519 ms 2024-09-13T22:35:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:42 smithi118 ceph-mon[51566]: pgmap v39: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.3 MiB/s wr, 707 op/s 2024-09-13T22:35:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:42 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:42 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:42 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:35:43.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 143640 9.73 MB/sec execute 251 sec latency 92.822 ms 2024-09-13T22:35:44.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 144227 9.75 MB/sec execute 252 sec latency 27.942 ms 2024-09-13T22:35:44.558 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:44 smithi105 ceph-mon[65554]: pgmap v40: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 487 op/s 2024-09-13T22:35:44.558 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:35:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:44 smithi118 ceph-mon[51566]: pgmap v40: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 487 op/s 2024-09-13T22:35:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:35:45.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 144594 9.77 MB/sec execute 253 sec latency 40.184 ms 2024-09-13T22:35:46.366 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:46 smithi105 ceph-mon[65554]: pgmap v41: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 646 op/s 2024-09-13T22:35:46.366 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:46.366 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:46.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 144813 9.76 MB/sec execute 254 sec latency 67.905 ms 2024-09-13T22:35:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:46 smithi118 ceph-mon[51566]: pgmap v41: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 646 op/s 2024-09-13T22:35:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:47.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 145092 9.75 MB/sec execute 255 sec latency 67.327 ms 2024-09-13T22:35:47.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:47.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:47.578 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:47 smithi105 ceph-mon[65554]: pgmap v42: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.2 MiB/s wr, 687 op/s 2024-09-13T22:35:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:47.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:47 smithi118 ceph-mon[51566]: pgmap v42: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.2 MiB/s wr, 687 op/s 2024-09-13T22:35:48.554 INFO:teuthology.orchestra.run.smithi105.stdout: 5 145351 9.74 MB/sec execute 256 sec latency 113.252 ms 2024-09-13T22:35:49.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 145603 9.72 MB/sec execute 257 sec latency 70.767 ms 2024-09-13T22:35:49.909 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:49 smithi105 ceph-mon[65554]: pgmap v43: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 578 op/s 2024-09-13T22:35:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:49 smithi118 ceph-mon[51566]: pgmap v43: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 578 op/s 2024-09-13T22:35:50.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 145789 9.70 MB/sec execute 258 sec latency 59.366 ms 2024-09-13T22:35:51.223 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:51 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:51.223 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:51 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:51.223 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:51 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:51.223 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:51 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:51.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:51 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:51.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:51 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:51.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:51 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:51.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:51 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:51.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 146050 9.70 MB/sec execute 259 sec latency 60.650 ms 2024-09-13T22:35:52.247 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:52 smithi105 ceph-mon[65554]: pgmap v44: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-09-13T22:35:52.247 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:52.247 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:52 smithi118 ceph-mon[51566]: pgmap v44: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-09-13T22:35:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:52.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 146296 9.68 MB/sec execute 260 sec latency 64.124 ms 2024-09-13T22:35:53.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 146613 9.66 MB/sec execute 261 sec latency 55.882 ms 2024-09-13T22:35:54.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:54 smithi105 ceph-mon[65554]: pgmap v45: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 450 op/s 2024-09-13T22:35:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:54 smithi118 ceph-mon[51566]: pgmap v45: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 450 op/s 2024-09-13T22:35:54.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 146962 9.67 MB/sec execute 262 sec latency 48.468 ms 2024-09-13T22:35:55.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 147287 9.66 MB/sec execute 263 sec latency 51.687 ms 2024-09-13T22:35:56.309 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:56 smithi105 ceph-mon[65554]: pgmap v46: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 604 op/s 2024-09-13T22:35:56.346 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:56 smithi118 ceph-mon[51566]: pgmap v46: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 604 op/s 2024-09-13T22:35:56.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 147727 9.66 MB/sec execute 264 sec latency 76.020 ms 2024-09-13T22:35:57.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 148088 9.66 MB/sec execute 265 sec latency 49.502 ms 2024-09-13T22:35:58.104 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: pgmap v47: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 475 op/s 2024-09-13T22:35:58.104 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.104 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all crash 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi105"}]: dispatch 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi105"}]': finished 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi118"}]: dispatch 2024-09-13T22:35:58.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi118"}]': finished 2024-09-13T22:35:58.106 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:35:58.106 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:35:58.106 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:58 smithi105 ceph-mon[65554]: Upgrade: osd.0 is safe to restart 2024-09-13T22:35:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: pgmap v47: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 475 op/s 2024-09-13T22:35:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:35:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:35:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:35:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:35:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all crash 2024-09-13T22:35:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:35:58.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi105"}]: dispatch 2024-09-13T22:35:58.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi105"}]': finished 2024-09-13T22:35:58.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi118"}]: dispatch 2024-09-13T22:35:58.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi118"}]': finished 2024-09-13T22:35:58.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:35:58.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T22:35:58.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:58 smithi118 ceph-mon[51566]: Upgrade: osd.0 is safe to restart 2024-09-13T22:35:58.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 148592 9.71 MB/sec execute 266 sec latency 42.114 ms 2024-09-13T22:35:59.114 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:35:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:35:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:35:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:35:59.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 149097 9.71 MB/sec execute 267 sec latency 43.888 ms 2024-09-13T22:36:00.139 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:00 smithi105 ceph-mon[65554]: pgmap v48: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.5 MiB/s wr, 344 op/s 2024-09-13T22:36:00.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:00.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:36:00.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:36:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:00 smithi118 ceph-mon[51566]: pgmap v48: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.5 MiB/s wr, 344 op/s 2024-09-13T22:36:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T22:36:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:36:00.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 149338 9.69 MB/sec execute 268 sec latency 42.788 ms 2024-09-13T22:36:01.173 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:01 smithi105 ceph-mon[65554]: Upgrade: Updating osd.0 2024-09-13T22:36:01.173 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:01 smithi105 ceph-mon[65554]: Deploying daemon osd.0 on smithi105 2024-09-13T22:36:01.555 INFO:teuthology.orchestra.run.smithi105.stdout: 5 149662 9.69 MB/sec execute 269 sec latency 54.018 ms 2024-09-13T22:36:01.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:01 smithi118 ceph-mon[51566]: Upgrade: Updating osd.0 2024-09-13T22:36:01.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:01 smithi118 ceph-mon[51566]: Deploying daemon osd.0 on smithi105 2024-09-13T22:36:02.197 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:02 smithi105 ceph-mon[65554]: pgmap v49: 97 pgs: 97 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.0 MiB/s wr, 589 op/s 2024-09-13T22:36:02.197 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:02 smithi105 ceph-mon[65554]: mgrmap e43: smithi105.swmmea(active, since 92s), standbys: smithi118.rbhtgz 2024-09-13T22:36:02.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 150048 9.68 MB/sec execute 270 sec latency 61.621 ms 2024-09-13T22:36:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:02 smithi118 ceph-mon[51566]: pgmap v49: 97 pgs: 97 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.0 MiB/s wr, 589 op/s 2024-09-13T22:36:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:02 smithi118 ceph-mon[51566]: mgrmap e43: smithi105.swmmea(active, since 92s), standbys: smithi118.rbhtgz 2024-09-13T22:36:03.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 150338 9.67 MB/sec execute 271 sec latency 57.448 ms 2024-09-13T22:36:04.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:04 smithi105 ceph-mon[65554]: pgmap v50: 97 pgs: 97 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-09-13T22:36:04.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 150669 9.68 MB/sec execute 272 sec latency 54.347 ms 2024-09-13T22:36:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:04 smithi118 ceph-mon[51566]: pgmap v50: 97 pgs: 97 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-09-13T22:36:05.060 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:05 smithi105 systemd[1]: Stopping Ceph osd.0 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:36:05.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 150991 9.67 MB/sec execute 273 sec latency 69.846 ms 2024-09-13T22:36:05.567 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:05 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[34217]: 2024-09-13T22:36:05.433+0000 7f9b134a3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:36:05.567 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:05 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[34217]: 2024-09-13T22:36:05.433+0000 7f9b134a3700 -1 osd.0 74 *** Got signal Terminated *** 2024-09-13T22:36:05.567 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:05 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[34217]: 2024-09-13T22:36:05.433+0000 7f9b134a3700 -1 osd.0 74 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:36:06.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:06 smithi105 ceph-mon[65554]: pgmap v51: 97 pgs: 97 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.1 MiB/s wr, 514 op/s 2024-09-13T22:36:06.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:06 smithi105 ceph-mon[65554]: osd.0 marked itself down and dead 2024-09-13T22:36:06.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 151190 9.66 MB/sec execute 274 sec latency 474.088 ms 2024-09-13T22:36:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:06 smithi118 ceph-mon[51566]: pgmap v51: 97 pgs: 97 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.1 MiB/s wr, 514 op/s 2024-09-13T22:36:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:06 smithi118 ceph-mon[51566]: osd.0 marked itself down and dead 2024-09-13T22:36:07.129 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:06 smithi105 podman[71116]: 2024-09-13 22:36:06.816811262 +0000 UTC m=+1.556764652 container died 3a67a5f434d288191d236ab7f797a284693665648263efc5b709c5bd38ff5713 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, distribution-scope=public, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, build-date=2022-05-03T08:36:31.336870, version=8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-13T22:36:07.387 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:07 smithi105 ceph-mon[65554]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:36:07.387 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:07 smithi105 ceph-mon[65554]: osdmap e75: 8 total, 7 up, 8 in 2024-09-13T22:36:07.387 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:07 smithi105 ceph-mon[65554]: osdmap e76: 8 total, 7 up, 8 in 2024-09-13T22:36:07.388 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:07 smithi105 podman[71116]: 2024-09-13 22:36:07.248840795 +0000 UTC m=+1.988794196 container cleanup 3a67a5f434d288191d236ab7f797a284693665648263efc5b709c5bd38ff5713 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0, architecture=x86_64, com.redhat.component=centos-stream-container, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, vendor=Red Hat, Inc.) 2024-09-13T22:36:07.388 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:07 smithi105 bash[71116]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0 2024-09-13T22:36:07.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 151395 9.64 MB/sec execute 275 sec latency 1064.980 ms 2024-09-13T22:36:07.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:07 smithi118 ceph-mon[51566]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:36:07.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:07 smithi118 ceph-mon[51566]: osdmap e75: 8 total, 7 up, 8 in 2024-09-13T22:36:07.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:07 smithi118 ceph-mon[51566]: osdmap e76: 8 total, 7 up, 8 in 2024-09-13T22:36:07.638 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:07 smithi105 podman[71130]: 2024-09-13 22:36:07.534382854 +0000 UTC m=+0.719718624 container remove 3a67a5f434d288191d236ab7f797a284693665648263efc5b709c5bd38ff5713 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., name=centos-stream, io.openshift.expose-services=, release=754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, version=8, distribution-scope=public, RELEASE=HEAD) 2024-09-13T22:36:08.153 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:08 smithi105 podman[71216]: 2024-09-13 22:36:07.930125774 +0000 UTC m=+0.019336538 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:36:08.153 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:08 smithi105 podman[71216]: 2024-09-13 22:36:08.043851632 +0000 UTC m=+0.133062395 container create 69bfd01472ba042af778ba88118a54839dd91a4249d33615164abc0caedcabbe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:36:08.406 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:08 smithi105 ceph-mon[65554]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 542 op/s 2024-09-13T22:36:08.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 151693 9.65 MB/sec execute 276 sec latency 86.801 ms 2024-09-13T22:36:08.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:08 smithi118 ceph-mon[51566]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 542 op/s 2024-09-13T22:36:08.657 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:08 smithi105 podman[71216]: 2024-09-13 22:36:08.60503835 +0000 UTC m=+0.694249129 container init 69bfd01472ba042af778ba88118a54839dd91a4249d33615164abc0caedcabbe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-13T22:36:08.657 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:08 smithi105 podman[71216]: 2024-09-13 22:36:08.617359219 +0000 UTC m=+0.706569991 container start 69bfd01472ba042af778ba88118a54839dd91a4249d33615164abc0caedcabbe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T22:36:08.910 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:08 smithi105 podman[71216]: 2024-09-13 22:36:08.795009506 +0000 UTC m=+0.884220280 container attach 69bfd01472ba042af778ba88118a54839dd91a4249d33615164abc0caedcabbe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T22:36:08.910 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:08 smithi105 podman[71216]: 2024-09-13 22:36:08.875789003 +0000 UTC m=+0.964999775 container died 69bfd01472ba042af778ba88118a54839dd91a4249d33615164abc0caedcabbe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T22:36:09.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 151936 9.63 MB/sec execute 277 sec latency 64.161 ms 2024-09-13T22:36:09.765 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:36:10.173 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:10 smithi105 podman[71239]: 2024-09-13 22:36:10.038122058 +0000 UTC m=+1.155781210 container remove 69bfd01472ba042af778ba88118a54839dd91a4249d33615164abc0caedcabbe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T22:36:10.173 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:10 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T22:36:10.174 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:10 smithi105 systemd[1]: Stopped Ceph osd.0 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:36:10.174 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:10 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 30.654s CPU time. 2024-09-13T22:36:10.174 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:10 smithi105 ceph-mon[65554]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.9 MiB/s wr, 311 op/s 2024-09-13T22:36:10.425 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:10 smithi105 systemd[1]: Starting Ceph osd.0 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:36:10.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 152259 9.64 MB/sec execute 278 sec latency 59.099 ms 2024-09-13T22:36:10.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:10 smithi118 ceph-mon[51566]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.9 MiB/s wr, 311 op/s 2024-09-13T22:36:10.678 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:10 smithi105 podman[71403]: 2024-09-13 22:36:10.572944083 +0000 UTC m=+0.021267695 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:36:10.753 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:36:10.753 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (4m) 19s ago 13m 18.8M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:36:10.753 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (35s) 19s ago 13m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:36:10.753 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (29s) 24s ago 12m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:36:10.753 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (3m) 19s ago 12m 46.6M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:36:10.753 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (7m) 19s ago 7m 37.8M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:36:10.753 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (7m) 24s ago 7m 14.3M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:36:10.753 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (3m) 19s ago 14m 496M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (2m) 24s ago 12m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (117s) 19s ago 14m 43.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (84s) 24s ago 12m 36.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (6m) 19s ago 6m 265M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (6m) 24s ago 6m 81.1M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (3m) 19s ago 12m 8514k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (3m) 24s ago 12m 14.5M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (11m) 19s ago 11m 527M 1936M 17.2.0 e1d6a67b021e 3a67a5f434d2 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (11m) 19s ago 11m 447M 1936M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (10m) 19s ago 10m 317M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (10m) 19s ago 10m 492M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (9m) 24s ago 9m 496M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (9m) 24s ago 9m 551M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:36:10.754 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (9m) 24s ago 9m 479M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:36:10.755 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (8m) 24s ago 8m 381M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:36:10.755 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (2m) 19s ago 12m 58.2M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:36:10.935 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:10 smithi105 podman[71403]: 2024-09-13 22:36:10.72856172 +0000 UTC m=+0.176885329 container create 07e86f3974e75a0d188fbb367eeb773a450d826bf2422e4d10c44e49664bc2f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate, GIT_CLEAN=True, RELEASE=squid-ecab7be, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T22:36:11.186 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:11 smithi105 podman[71403]: 2024-09-13 22:36:11.102215417 +0000 UTC m=+0.550539039 container init 07e86f3974e75a0d188fbb367eeb773a450d826bf2422e4d10c44e49664bc2f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-13T22:36:11.186 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:11 smithi105 podman[71403]: 2024-09-13 22:36:11.114623364 +0000 UTC m=+0.562946981 container start 07e86f3974e75a0d188fbb367eeb773a450d826bf2422e4d10c44e49664bc2f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:36:11.186 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:11 smithi105 ceph-mon[65554]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:11.186 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:11 smithi105 ceph-mon[65554]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:11.280 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:36:11.280 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:36:11.280 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T22:36:11.281 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:36:11.282 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:36:11.441 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:11 smithi105 podman[71403]: 2024-09-13 22:36:11.229097769 +0000 UTC m=+0.677421386 container attach 07e86f3974e75a0d188fbb367eeb773a450d826bf2422e4d10c44e49664bc2f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-13T22:36:11.441 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:11 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate[71441]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:36:11.441 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:11 smithi105 bash[71403]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:36:11.441 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:11 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate[71441]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:36:11.441 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:11 smithi105 bash[71403]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:36:11.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 152621 9.64 MB/sec execute 279 sec latency 49.648 ms 2024-09-13T22:36:11.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:11 smithi118 ceph-mon[51566]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:11.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:11 smithi118 ceph-mon[51566]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:11.845 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:36:11.845 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:36:11.845 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "6/23 daemons upgraded", 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:36:11.846 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:36:12.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:12 smithi105 ceph-mon[65554]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:12.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:12 smithi105 ceph-mon[65554]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.5 MiB/s wr, 535 op/s; 223/1734 objects degraded (12.860%) 2024-09-13T22:36:12.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:12 smithi105 ceph-mon[65554]: Health check failed: Degraded data redundancy: 223/1734 objects degraded (12.860%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:12.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:12 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/3506855741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:12.367 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 223/1734 objects degraded (12.860%), 29 pgs degraded 2024-09-13T22:36:12.367 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T22:36:12.367 INFO:teuthology.orchestra.run.smithi105.stdout: osd.0 (root=default,host=smithi105) is down 2024-09-13T22:36:12.367 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 223/1734 objects degraded (12.860%), 29 pgs degraded 2024-09-13T22:36:12.367 INFO:teuthology.orchestra.run.smithi105.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-13T22:36:12.367 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-13T22:36:12.367 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-13T22:36:12.367 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1f is active+undersized+degraded, acting [2,4] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-13T22:36:12.368 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-13T22:36:12.369 INFO:teuthology.orchestra.run.smithi105.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-13T22:36:12.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 152855 9.62 MB/sec execute 280 sec latency 50.191 ms 2024-09-13T22:36:12.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:12 smithi118 ceph-mon[51566]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:12.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:12 smithi118 ceph-mon[51566]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.5 MiB/s wr, 535 op/s; 223/1734 objects degraded (12.860%) 2024-09-13T22:36:12.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:12 smithi118 ceph-mon[51566]: Health check failed: Degraded data redundancy: 223/1734 objects degraded (12.860%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:12.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:12 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/3506855741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:12.991 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:12 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate[71441]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:36:12.991 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:12 smithi105 bash[71403]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:36:12.991 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:12 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate[71441]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T22:36:12.991 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:12 smithi105 bash[71403]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T22:36:13.267 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate[71441]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T22:36:13.267 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 bash[71403]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T22:36:13.267 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate[71441]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T22:36:13.268 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 bash[71403]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T22:36:13.268 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate[71441]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T22:36:13.268 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 bash[71403]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T22:36:13.268 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate[71441]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:36:13.268 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 bash[71403]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T22:36:13.268 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate[71441]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T22:36:13.268 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 bash[71403]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T22:36:13.268 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:13 smithi105 podman[71403]: 2024-09-13 22:36:13.241385777 +0000 UTC m=+2.689709395 container died 07e86f3974e75a0d188fbb367eeb773a450d826bf2422e4d10c44e49664bc2f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux ) 2024-09-13T22:36:13.268 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:13 smithi105 ceph-mon[65554]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:13.269 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:13 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/475830496' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:36:13.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 153147 9.61 MB/sec execute 281 sec latency 44.549 ms 2024-09-13T22:36:13.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:13 smithi118 ceph-mon[51566]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:13.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:13 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/475830496' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:36:14.276 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:14 smithi105 ceph-mon[65554]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 223/1734 objects degraded (12.860%) 2024-09-13T22:36:14.276 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:14.276 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:36:14.529 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:14 smithi105 podman[71638]: 2024-09-13 22:36:14.478012669 +0000 UTC m=+1.230818197 container remove 07e86f3974e75a0d188fbb367eeb773a450d826bf2422e4d10c44e49664bc2f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T22:36:14.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 153633 9.61 MB/sec execute 282 sec latency 43.411 ms 2024-09-13T22:36:14.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:14 smithi118 ceph-mon[51566]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 223/1734 objects degraded (12.860%) 2024-09-13T22:36:14.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:14.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:36:14.780 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:14 smithi105 podman[71692]: 2024-09-13 22:36:14.638180632 +0000 UTC m=+0.021929972 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:36:14.780 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:14 smithi105 podman[71692]: 2024-09-13 22:36:14.761396036 +0000 UTC m=+0.145145380 container create d259261fb822184f8dd37566365c660016e6de12589576eea60835b56f33a580 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:36:15.288 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:15 smithi105 podman[71692]: 2024-09-13 22:36:15.209338037 +0000 UTC m=+0.593087399 container init d259261fb822184f8dd37566365c660016e6de12589576eea60835b56f33a580 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731) 2024-09-13T22:36:15.288 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:15 smithi105 podman[71692]: 2024-09-13 22:36:15.2194801 +0000 UTC m=+0.603229448 container start d259261fb822184f8dd37566365c660016e6de12589576eea60835b56f33a580 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be) 2024-09-13T22:36:15.289 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:15 smithi105 ceph-osd[71711]: -- 172.21.15.105:0/750090418 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558178c40000 con 0x558177e51c00 2024-09-13T22:36:15.539 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:15 smithi105 bash[71692]: d259261fb822184f8dd37566365c660016e6de12589576eea60835b56f33a580 2024-09-13T22:36:15.539 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:15 smithi105 systemd[1]: Started Ceph osd.0 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:36:15.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 154109 9.64 MB/sec execute 283 sec latency 44.044 ms 2024-09-13T22:36:16.301 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:16 smithi105 ceph-mon[65554]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 4.5 MiB/s wr, 427 op/s; 217/1710 objects degraded (12.690%) 2024-09-13T22:36:16.301 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:16.301 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:16.301 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:36:16.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 154552 9.64 MB/sec execute 284 sec latency 32.943 ms 2024-09-13T22:36:16.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:16 smithi118 ceph-mon[51566]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 4.5 MiB/s wr, 427 op/s; 217/1710 objects degraded (12.690%) 2024-09-13T22:36:16.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:16.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:16.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:36:17.326 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:17 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 217/1716 objects degraded (12.646%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:17.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 154994 9.64 MB/sec execute 285 sec latency 42.606 ms 2024-09-13T22:36:17.581 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:17 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[71707]: 2024-09-13T22:36:17.558+0000 7fd6370c5640 -1 Falling back to public interface 2024-09-13T22:36:17.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:17 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 217/1716 objects degraded (12.646%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:18.341 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:18 smithi105 ceph-mon[65554]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 433 op/s; 217/1716 objects degraded (12.646%) 2024-09-13T22:36:18.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 155269 9.64 MB/sec execute 286 sec latency 58.392 ms 2024-09-13T22:36:18.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:18 smithi118 ceph-mon[51566]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 433 op/s; 217/1716 objects degraded (12.646%) 2024-09-13T22:36:19.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 155596 9.64 MB/sec execute 287 sec latency 52.181 ms 2024-09-13T22:36:20.386 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:20 smithi105 ceph-mon[65554]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 370 op/s; 217/1716 objects degraded (12.646%) 2024-09-13T22:36:20.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 155927 9.65 MB/sec execute 288 sec latency 64.506 ms 2024-09-13T22:36:20.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:20 smithi118 ceph-mon[51566]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 370 op/s; 217/1716 objects degraded (12.646%) 2024-09-13T22:36:21.557 INFO:teuthology.orchestra.run.smithi105.stdout: 5 156303 9.64 MB/sec execute 289 sec latency 53.436 ms 2024-09-13T22:36:22.495 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:22 smithi105 ceph-mon[65554]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 588 op/s; 220/1692 objects degraded (13.002%) 2024-09-13T22:36:22.557 INFO:teuthology.orchestra.run.smithi105.stdout: 5 156543 9.63 MB/sec execute 290 sec latency 48.915 ms 2024-09-13T22:36:22.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:22 smithi118 ceph-mon[51566]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 588 op/s; 220/1692 objects degraded (13.002%) 2024-09-13T22:36:23.557 INFO:teuthology.orchestra.run.smithi105.stdout: 5 156889 9.62 MB/sec execute 291 sec latency 51.187 ms 2024-09-13T22:36:24.268 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:24 smithi105 ceph-mon[65554]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 438 op/s; 220/1692 objects degraded (13.002%) 2024-09-13T22:36:24.268 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:24 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:24.268 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:24 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:24.268 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:24 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 220/1692 objects degraded (13.002%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:24.557 INFO:teuthology.orchestra.run.smithi105.stdout: 5 157233 9.62 MB/sec execute 292 sec latency 65.887 ms 2024-09-13T22:36:24.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:24 smithi118 ceph-mon[51566]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 438 op/s; 220/1692 objects degraded (13.002%) 2024-09-13T22:36:24.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:24 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:24.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:24 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:24.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:24 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 220/1692 objects degraded (13.002%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:25.042 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:24 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[71707]: 2024-09-13T22:36:24.787+0000 7fd6370c5640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-13T22:36:25.556 INFO:teuthology.orchestra.run.smithi105.stdout: 5 157544 9.61 MB/sec execute 293 sec latency 46.227 ms 2024-09-13T22:36:25.817 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:25 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[71707]: 2024-09-13T22:36:25.660+0000 7fd6370c5640 -1 osd.0 74 log_to_monitors true 2024-09-13T22:36:25.817 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:25.817 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:25.817 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:25 smithi105 ceph-mon[65554]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 522 op/s; 223/1716 objects degraded (12.995%) 2024-09-13T22:36:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:25 smithi118 ceph-mon[51566]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 522 op/s; 223/1716 objects degraded (12.995%) 2024-09-13T22:36:26.557 INFO:teuthology.orchestra.run.smithi105.stdout: 5 157814 9.61 MB/sec execute 294 sec latency 60.465 ms 2024-09-13T22:36:26.831 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:36:26 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[71707]: 2024-09-13T22:36:26.725+0000 7fd62ee70640 -1 osd.0 74 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:36:26.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:26 smithi105 ceph-mon[65554]: from='osd.0 [v2:172.21.15.105:6802/3032770746,v1:172.21.15.105:6803/3032770746]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:36:26.831 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:26 smithi105 ceph-mon[65554]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:36:27.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:26 smithi118 ceph-mon[51566]: from='osd.0 [v2:172.21.15.105:6802/3032770746,v1:172.21.15.105:6803/3032770746]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:36:27.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:26 smithi118 ceph-mon[51566]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T22:36:27.557 INFO:teuthology.orchestra.run.smithi105.stdout: 5 158214 9.61 MB/sec execute 295 sec latency 48.206 ms 2024-09-13T22:36:27.836 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:27 smithi105 ceph-mon[65554]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:36:27.836 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:27 smithi105 ceph-mon[65554]: osdmap e77: 8 total, 7 up, 8 in 2024-09-13T22:36:27.836 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:27 smithi105 ceph-mon[65554]: from='osd.0 [v2:172.21.15.105:6802/3032770746,v1:172.21.15.105:6803/3032770746]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:36:27.836 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:27 smithi105 ceph-mon[65554]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:36:27.836 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:27 smithi105 ceph-mon[65554]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 520 op/s; 226/1704 objects degraded (13.263%) 2024-09-13T22:36:28.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:27 smithi118 ceph-mon[51566]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T22:36:28.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:27 smithi118 ceph-mon[51566]: osdmap e77: 8 total, 7 up, 8 in 2024-09-13T22:36:28.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:27 smithi118 ceph-mon[51566]: from='osd.0 [v2:172.21.15.105:6802/3032770746,v1:172.21.15.105:6803/3032770746]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:36:28.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:27 smithi118 ceph-mon[51566]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:36:28.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:27 smithi118 ceph-mon[51566]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 520 op/s; 226/1704 objects degraded (13.263%) 2024-09-13T22:36:28.558 INFO:teuthology.orchestra.run.smithi105.stdout: 5 158390 9.60 MB/sec execute 296 sec latency 866.415 ms 2024-09-13T22:36:28.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:28 smithi105 ceph-mon[65554]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:36:28.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:28 smithi105 ceph-mon[65554]: osd.0 [v2:172.21.15.105:6802/3032770746,v1:172.21.15.105:6803/3032770746] boot 2024-09-13T22:36:28.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:28 smithi105 ceph-mon[65554]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T22:36:28.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:36:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:28 smithi118 ceph-mon[51566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:36:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:28 smithi118 ceph-mon[51566]: osd.0 [v2:172.21.15.105:6802/3032770746,v1:172.21.15.105:6803/3032770746] boot 2024-09-13T22:36:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:28 smithi118 ceph-mon[51566]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T22:36:29.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T22:36:29.558 INFO:teuthology.orchestra.run.smithi105.stdout: 5 158390 9.56 MB/sec execute 297 sec latency 1866.468 ms 2024-09-13T22:36:29.935 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T22:36:29.935 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 414 op/s; 226/1704 objects degraded (13.263%) 2024-09-13T22:36:29.935 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:29.935 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 226/1704 objects degraded (13.263%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:29.936 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T22:36:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T22:36:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 414 op/s; 226/1704 objects degraded (13.263%) 2024-09-13T22:36:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:36:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 226/1704 objects degraded (13.263%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:30.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:30.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T22:36:30.558 INFO:teuthology.orchestra.run.smithi105.stdout: 5 158667 9.55 MB/sec execute 298 sec latency 2074.938 ms 2024-09-13T22:36:30.697 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:30 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T22:36:30.697 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:30 smithi105 ceph-mon[65554]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T22:36:30.697 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:30 smithi105 ceph-mon[65554]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T22:36:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:30 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T22:36:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:30 smithi118 ceph-mon[51566]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T22:36:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:30 smithi118 ceph-mon[51566]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T22:36:31.558 INFO:teuthology.orchestra.run.smithi105.stdout: 5 158982 9.56 MB/sec execute 299 sec latency 485.218 ms 2024-09-13T22:36:31.718 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:31 smithi105 ceph-mon[65554]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T22:36:31.718 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:31 smithi105 ceph-mon[65554]: pgmap v71: 97 pgs: 1 active+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized, 1 active+undersized+degraded, 78 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 400 op/s; 254/1578 objects degraded (16.096%); 1.2 MiB/s, 10 objects/s recovering 2024-09-13T22:36:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:31 smithi118 ceph-mon[51566]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T22:36:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:31 smithi118 ceph-mon[51566]: pgmap v71: 97 pgs: 1 active+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized, 1 active+undersized+degraded, 78 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 400 op/s; 254/1578 objects degraded (16.096%); 1.2 MiB/s, 10 objects/s recovering 2024-09-13T22:36:32.558 INFO:teuthology.orchestra.run.smithi105.stdout: 5 159277 9.55 MB/sec execute 300 sec latency 644.014 ms 2024-09-13T22:36:32.723 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:32 smithi105 ceph-mon[65554]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T22:36:33.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:32 smithi118 ceph-mon[51566]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T22:36:33.558 INFO:teuthology.orchestra.run.smithi105.stdout: 5 159713 9.57 MB/sec execute 301 sec latency 419.841 ms 2024-09-13T22:36:33.742 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:33 smithi105 ceph-mon[65554]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T22:36:33.742 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:33 smithi105 ceph-mon[65554]: pgmap v74: 97 pgs: 3 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 404 op/s; 203/1560 objects degraded (13.013%); 1.2 MiB/s, 21 objects/s recovering 2024-09-13T22:36:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:33 smithi118 ceph-mon[51566]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T22:36:34.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:33 smithi118 ceph-mon[51566]: pgmap v74: 97 pgs: 3 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 404 op/s; 203/1560 objects degraded (13.013%); 1.2 MiB/s, 21 objects/s recovering 2024-09-13T22:36:34.558 INFO:teuthology.orchestra.run.smithi105.stdout: 5 159807 9.54 MB/sec execute 302 sec latency 827.306 ms 2024-09-13T22:36:34.998 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:34 smithi105 ceph-mon[65554]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T22:36:34.998 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:34 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 203/1560 objects degraded (13.013%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:34 smithi118 ceph-mon[51566]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T22:36:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:34 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 203/1560 objects degraded (13.013%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:35.558 INFO:teuthology.orchestra.run.smithi105.stdout: 5 160190 9.53 MB/sec execute 303 sec latency 1017.025 ms 2024-09-13T22:36:35.757 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:35 smithi105 ceph-mon[65554]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T22:36:35.757 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:35 smithi105 ceph-mon[65554]: pgmap v77: 97 pgs: 3 peering, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 MiB/s wr, 505 op/s; 102/1620 objects degraded (6.296%); 1023 KiB/s, 36 objects/s recovering 2024-09-13T22:36:36.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:35 smithi118 ceph-mon[51566]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T22:36:36.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:35 smithi118 ceph-mon[51566]: pgmap v77: 97 pgs: 3 peering, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 MiB/s wr, 505 op/s; 102/1620 objects degraded (6.296%); 1023 KiB/s, 36 objects/s recovering 2024-09-13T22:36:36.559 INFO:teuthology.orchestra.run.smithi105.stdout: 5 160562 9.54 MB/sec execute 304 sec latency 60.695 ms 2024-09-13T22:36:37.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:36 smithi105 ceph-mon[65554]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 102/1620 objects degraded (6.296%), 4 pgs degraded) 2024-09-13T22:36:37.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:36 smithi105 ceph-mon[65554]: Cluster is now healthy 2024-09-13T22:36:37.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:36 smithi118 ceph-mon[51566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 102/1620 objects degraded (6.296%), 4 pgs degraded) 2024-09-13T22:36:37.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:36 smithi118 ceph-mon[51566]: Cluster is now healthy 2024-09-13T22:36:37.558 INFO:teuthology.orchestra.run.smithi105.stdout: 5 160946 9.54 MB/sec execute 305 sec latency 49.747 ms 2024-09-13T22:36:38.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:37 smithi105 ceph-mon[65554]: pgmap v78: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 429 op/s; 1.6 MiB/s, 39 objects/s recovering 2024-09-13T22:36:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:37 smithi118 ceph-mon[51566]: pgmap v78: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 429 op/s; 1.6 MiB/s, 39 objects/s recovering 2024-09-13T22:36:38.566 INFO:teuthology.orchestra.run.smithi105.stdout: 5 161302 9.54 MB/sec execute 306 sec latency 63.992 ms 2024-09-13T22:36:39.566 INFO:teuthology.orchestra.run.smithi105.stdout: 5 161661 9.55 MB/sec execute 307 sec latency 47.961 ms 2024-09-13T22:36:40.123 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:39 smithi105 ceph-mon[65554]: pgmap v79: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.7 MiB/s wr, 359 op/s; 1.3 MiB/s, 33 objects/s recovering 2024-09-13T22:36:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:39 smithi118 ceph-mon[51566]: pgmap v79: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.7 MiB/s wr, 359 op/s; 1.3 MiB/s, 33 objects/s recovering 2024-09-13T22:36:40.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 161924 9.54 MB/sec execute 308 sec latency 81.703 ms 2024-09-13T22:36:41.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 162235 9.53 MB/sec execute 309 sec latency 54.988 ms 2024-09-13T22:36:42.196 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:41 smithi105 ceph-mon[65554]: pgmap v80: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 585 op/s; 1023 KiB/s, 25 objects/s recovering 2024-09-13T22:36:42.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:41 smithi118 ceph-mon[51566]: pgmap v80: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 585 op/s; 1023 KiB/s, 25 objects/s recovering 2024-09-13T22:36:42.566 INFO:teuthology.orchestra.run.smithi105.stdout: 5 162527 9.53 MB/sec execute 310 sec latency 56.874 ms 2024-09-13T22:36:42.896 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:36:43.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 162788 9.52 MB/sec execute 311 sec latency 48.408 ms 2024-09-13T22:36:43.972 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:43 smithi105 ceph-mon[65554]: pgmap v81: 97 pgs: 97 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 537 op/s; 898 KiB/s, 22 objects/s recovering 2024-09-13T22:36:43.973 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:43 smithi105 ceph-mon[65554]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:43.973 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:43 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:43.973 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:43 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:43.973 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:43 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:36:43.997 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:36:43.997 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (4m) 20s ago 13m 18.8M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:36:43.997 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (68s) 20s ago 13m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:36:43.997 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (62s) 58s ago 13m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:36:43.997 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (4m) 20s ago 13m 45.8M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:36:43.997 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (7m) 20s ago 7m 37.8M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:36:43.997 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (7m) 58s ago 7m 14.3M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (3m) 20s ago 14m 502M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (2m) 58s ago 13m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (2m) 20s ago 15m 46.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (117s) 58s ago 13m 36.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (7m) 20s ago 7m 266M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (7m) 58s ago 7m 81.1M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (4m) 20s ago 13m 8250k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (3m) 58s ago 12m 14.5M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (29s) 20s ago 12m 11.7M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (11m) 20s ago 11m 480M 1936M 17.2.0 e1d6a67b021e 3b3896d8ffcf 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (11m) 20s ago 11m 350M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (10m) 20s ago 10m 507M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (10m) 58s ago 10m 496M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (10m) 58s ago 10m 551M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:36:43.998 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (9m) 58s ago 9m 479M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:36:43.999 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (9m) 58s ago 9m 381M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:36:43.999 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (2m) 20s ago 12m 60.7M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:36:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:43 smithi118 ceph-mon[51566]: pgmap v81: 97 pgs: 97 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 537 op/s; 898 KiB/s, 22 objects/s recovering 2024-09-13T22:36:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:43 smithi118 ceph-mon[51566]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:43 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:43 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:43 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:36:44.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 163090 9.52 MB/sec execute 312 sec latency 63.302 ms 2024-09-13T22:36:44.637 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:36:44.637 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:36:44.637 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:36:44.637 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:36:44.637 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:36:44.637 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:36:44.637 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:36:44.637 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 1 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:36:44.638 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:36:44.990 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:44 smithi105 ceph-mon[65554]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:44.990 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:44 smithi105 ceph-mon[65554]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:44.990 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T22:36:44.990 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:44 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2138006436' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:45.164 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:36:45.164 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:36:45.164 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:36:45.164 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:36:45.165 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:36:45.165 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:36:45.165 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:36:45.165 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:36:45.165 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:36:45.165 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "7/23 daemons upgraded", 2024-09-13T22:36:45.165 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:36:45.165 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:36:45.165 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:36:45.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:44 smithi118 ceph-mon[51566]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:45.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:44 smithi118 ceph-mon[51566]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:45.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T22:36:45.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:44 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2138006436' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:36:45.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 163380 9.51 MB/sec execute 313 sec latency 49.127 ms 2024-09-13T22:36:45.785 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:36:46.003 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:45 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T22:36:46.003 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:45 smithi105 ceph-mon[65554]: Upgrade: osd.1 is safe to restart 2024-09-13T22:36:46.003 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:45 smithi105 ceph-mon[65554]: pgmap v82: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.9 MiB/s wr, 685 op/s; 810 KiB/s, 20 objects/s recovering 2024-09-13T22:36:46.003 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:45 smithi105 ceph-mon[65554]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:46.004 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:45 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/1118488968' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:36:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:45 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T22:36:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:45 smithi118 ceph-mon[51566]: Upgrade: osd.1 is safe to restart 2024-09-13T22:36:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:45 smithi118 ceph-mon[51566]: pgmap v82: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.9 MiB/s wr, 685 op/s; 810 KiB/s, 20 objects/s recovering 2024-09-13T22:36:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:45 smithi118 ceph-mon[51566]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:36:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:45 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/1118488968' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:36:46.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 163672 9.50 MB/sec execute 314 sec latency 46.947 ms 2024-09-13T22:36:47.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 163909 9.49 MB/sec execute 315 sec latency 63.397 ms 2024-09-13T22:36:47.808 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:47 smithi105 ceph-mon[65554]: Upgrade: Updating osd.1 2024-09-13T22:36:47.808 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:47.808 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T22:36:47.808 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:36:47.808 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:47 smithi105 ceph-mon[65554]: Deploying daemon osd.1 on smithi105 2024-09-13T22:36:47.808 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:47 smithi105 ceph-mon[65554]: pgmap v83: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.4 MiB/s wr, 423 op/s; 341 KiB/s, 8 objects/s recovering 2024-09-13T22:36:48.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:47 smithi118 ceph-mon[51566]: Upgrade: Updating osd.1 2024-09-13T22:36:48.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:36:48.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T22:36:48.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:36:48.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:47 smithi118 ceph-mon[51566]: Deploying daemon osd.1 on smithi105 2024-09-13T22:36:48.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:47 smithi118 ceph-mon[51566]: pgmap v83: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.4 MiB/s wr, 423 op/s; 341 KiB/s, 8 objects/s recovering 2024-09-13T22:36:48.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 164150 9.48 MB/sec execute 316 sec latency 79.814 ms 2024-09-13T22:36:49.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 164406 9.47 MB/sec execute 317 sec latency 65.248 ms 2024-09-13T22:36:49.921 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:49 smithi105 ceph-mon[65554]: pgmap v84: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 407 op/s 2024-09-13T22:36:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:49 smithi118 ceph-mon[51566]: pgmap v84: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 407 op/s 2024-09-13T22:36:50.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 164661 9.47 MB/sec execute 318 sec latency 53.009 ms 2024-09-13T22:36:51.205 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:50 smithi105 systemd[1]: Stopping Ceph osd.1 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:36:51.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 164910 9.46 MB/sec execute 319 sec latency 65.899 ms 2024-09-13T22:36:51.726 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:51 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[37058]: 2024-09-13T22:36:51.493+0000 7fb0266b9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:36:51.726 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:51 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[37058]: 2024-09-13T22:36:51.493+0000 7fb0266b9700 -1 osd.1 85 *** Got signal Terminated *** 2024-09-13T22:36:51.726 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:51 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[37058]: 2024-09-13T22:36:51.493+0000 7fb0266b9700 -1 osd.1 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:36:52.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:51 smithi105 ceph-mon[65554]: pgmap v85: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 558 op/s 2024-09-13T22:36:52.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:51 smithi105 ceph-mon[65554]: osd.1 marked itself down and dead 2024-09-13T22:36:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:51 smithi118 ceph-mon[51566]: pgmap v85: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 558 op/s 2024-09-13T22:36:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:51 smithi118 ceph-mon[51566]: osd.1 marked itself down and dead 2024-09-13T22:36:52.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 165020 9.45 MB/sec execute 320 sec latency 674.558 ms 2024-09-13T22:36:52.890 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:52 smithi105 podman[73852]: 2024-09-13 22:36:52.585973262 +0000 UTC m=+1.411068385 container died 3b3896d8ffcf2e6eb8b1ebed4990edd124da9d2a09999779b58ff175596b7b0c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, name=centos-stream, io.openshift.expose-services=, vcs-type=git, com.redhat.component=centos-stream-container, architecture=x86_64, RELEASE=HEAD, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8) 2024-09-13T22:36:53.165 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:52 smithi105 ceph-mon[65554]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:36:53.165 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:52 smithi105 ceph-mon[65554]: osdmap e86: 8 total, 7 up, 8 in 2024-09-13T22:36:53.166 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:52 smithi105 podman[73852]: 2024-09-13 22:36:52.947138256 +0000 UTC m=+1.772233376 container cleanup 3b3896d8ffcf2e6eb8b1ebed4990edd124da9d2a09999779b58ff175596b7b0c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, ceph=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vendor=Red Hat, Inc., io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754) 2024-09-13T22:36:53.166 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:52 smithi105 bash[73852]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1 2024-09-13T22:36:53.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:52 smithi118 ceph-mon[51566]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:36:53.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:52 smithi118 ceph-mon[51566]: osdmap e86: 8 total, 7 up, 8 in 2024-09-13T22:36:53.385 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:53 smithi105 podman[73867]: 2024-09-13 22:36:53.16514845 +0000 UTC m=+0.581633859 container remove 3b3896d8ffcf2e6eb8b1ebed4990edd124da9d2a09999779b58ff175596b7b0c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, name=centos-stream, vcs-type=git, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64) 2024-09-13T22:36:53.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 165162 9.43 MB/sec execute 321 sec latency 1126.415 ms 2024-09-13T22:36:53.679 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:53 smithi105 podman[73954]: 2024-09-13 22:36:53.533356915 +0000 UTC m=+0.021637263 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:36:53.931 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:53 smithi105 podman[73954]: 2024-09-13 22:36:53.683180457 +0000 UTC m=+0.171460786 container create 6470ea832b9de3e58c22a8ba972b03009f177526ab32a838ae1f1d3db62a68d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_CLEAN=True) 2024-09-13T22:36:54.184 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:53 smithi105 ceph-mon[65554]: pgmap v87: 97 pgs: 13 stale+active+clean, 84 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.0 MiB/s wr, 411 op/s 2024-09-13T22:36:54.184 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:53 smithi105 ceph-mon[65554]: osdmap e87: 8 total, 7 up, 8 in 2024-09-13T22:36:54.184 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:54 smithi105 podman[73954]: 2024-09-13 22:36:54.053413897 +0000 UTC m=+0.541694221 container init 6470ea832b9de3e58c22a8ba972b03009f177526ab32a838ae1f1d3db62a68d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T22:36:54.184 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:54 smithi105 podman[73954]: 2024-09-13 22:36:54.065478737 +0000 UTC m=+0.553759069 container start 6470ea832b9de3e58c22a8ba972b03009f177526ab32a838ae1f1d3db62a68d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux ) 2024-09-13T22:36:54.185 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:54 smithi105 podman[73954]: 2024-09-13 22:36:54.153322785 +0000 UTC m=+0.641603115 container attach 6470ea832b9de3e58c22a8ba972b03009f177526ab32a838ae1f1d3db62a68d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T22:36:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:53 smithi118 ceph-mon[51566]: pgmap v87: 97 pgs: 13 stale+active+clean, 84 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.0 MiB/s wr, 411 op/s 2024-09-13T22:36:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:53 smithi118 ceph-mon[51566]: osdmap e87: 8 total, 7 up, 8 in 2024-09-13T22:36:54.436 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:54 smithi105 podman[73954]: 2024-09-13 22:36:54.391599708 +0000 UTC m=+0.879880040 container died 6470ea832b9de3e58c22a8ba972b03009f177526ab32a838ae1f1d3db62a68d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-13T22:36:54.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 165493 9.43 MB/sec execute 322 sec latency 52.229 ms 2024-09-13T22:36:55.195 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:54 smithi105 ceph-mon[65554]: Health check failed: Degraded data redundancy: 132/1635 objects degraded (8.073%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:55.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:54 smithi118 ceph-mon[51566]: Health check failed: Degraded data redundancy: 132/1635 objects degraded (8.073%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T22:36:55.447 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:55 smithi105 podman[73976]: 2024-09-13 22:36:55.403507259 +0000 UTC m=+0.995102655 container remove 6470ea832b9de3e58c22a8ba972b03009f177526ab32a838ae1f1d3db62a68d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T22:36:55.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 165851 9.42 MB/sec execute 323 sec latency 60.231 ms 2024-09-13T22:36:55.714 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:55 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T22:36:55.714 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:55 smithi105 systemd[1]: Stopped Ceph osd.1 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:36:55.714 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:55 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.1.service: Consumed 2min 34.031s CPU time. 2024-09-13T22:36:55.971 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:55 smithi105 systemd[1]: Starting Ceph osd.1 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:36:56.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:55 smithi105 ceph-mon[65554]: pgmap v89: 97 pgs: 12 active+undersized, 4 stale+active+clean, 18 active+undersized+degraded, 63 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.2 MiB/s wr, 422 op/s; 132/1635 objects degraded (8.073%) 2024-09-13T22:36:56.226 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:55 smithi105 podman[74073]: 2024-09-13 22:36:55.882381692 +0000 UTC m=+0.017719430 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:36:56.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:55 smithi118 ceph-mon[51566]: pgmap v89: 97 pgs: 12 active+undersized, 4 stale+active+clean, 18 active+undersized+degraded, 63 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.2 MiB/s wr, 422 op/s; 132/1635 objects degraded (8.073%) 2024-09-13T22:36:56.481 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:56 smithi105 podman[74073]: 2024-09-13 22:36:56.37117122 +0000 UTC m=+0.506508929 container create 40e37fc75f96d603f5b404fe9d137042f70f80d47e50f5680b88b33172982962 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=squid-ecab7be, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:36:56.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 166129 9.41 MB/sec execute 324 sec latency 57.849 ms 2024-09-13T22:36:56.917 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:56 smithi105 podman[74073]: 2024-09-13 22:36:56.858305142 +0000 UTC m=+0.993642862 container init 40e37fc75f96d603f5b404fe9d137042f70f80d47e50f5680b88b33172982962 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T22:36:56.918 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:56 smithi105 podman[74073]: 2024-09-13 22:36:56.872217447 +0000 UTC m=+1.007555164 container start 40e37fc75f96d603f5b404fe9d137042f70f80d47e50f5680b88b33172982962 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T22:36:57.247 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:57 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate[74088]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:36:57.247 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:57 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate[74088]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:36:57.247 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:57 smithi105 bash[74073]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:36:57.248 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:57 smithi105 bash[74073]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:36:57.248 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:57 smithi105 podman[74073]: 2024-09-13 22:36:57.051912749 +0000 UTC m=+1.187250471 container attach 40e37fc75f96d603f5b404fe9d137042f70f80d47e50f5680b88b33172982962 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T22:36:57.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 166369 9.41 MB/sec execute 325 sec latency 38.691 ms 2024-09-13T22:36:58.017 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:58 smithi105 ceph-mon[65554]: pgmap v90: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 439 op/s; 178/1647 objects degraded (10.808%) 2024-09-13T22:36:58.268 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate[74088]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T22:36:58.268 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 bash[74073]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T22:36:58.268 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate[74088]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T22:36:58.268 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 bash[74073]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T22:36:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:58 smithi118 ceph-mon[51566]: pgmap v90: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 439 op/s; 178/1647 objects degraded (10.808%) 2024-09-13T22:36:58.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 166705 9.42 MB/sec execute 326 sec latency 59.499 ms 2024-09-13T22:36:58.786 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate[74088]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T22:36:58.786 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 bash[74073]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T22:36:58.786 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate[74088]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T22:36:58.787 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 bash[74073]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T22:36:58.787 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate[74088]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T22:36:58.787 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 bash[74073]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T22:36:58.787 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate[74088]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T22:36:58.787 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 bash[74073]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T22:36:58.787 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate[74088]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T22:36:58.787 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 bash[74073]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T22:36:58.787 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:58 smithi105 podman[74073]: 2024-09-13 22:36:58.629261755 +0000 UTC m=+2.764599465 container died 40e37fc75f96d603f5b404fe9d137042f70f80d47e50f5680b88b33172982962 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T22:36:59.550 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:59 smithi105 podman[74227]: 2024-09-13 22:36:59.490096773 +0000 UTC m=+0.853337604 container remove 40e37fc75f96d603f5b404fe9d137042f70f80d47e50f5680b88b33172982962 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T22:36:59.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 167065 9.40 MB/sec execute 327 sec latency 47.204 ms 2024-09-13T22:36:59.801 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:59 smithi105 podman[74285]: 2024-09-13 22:36:59.662882742 +0000 UTC m=+0.022969412 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:37:00.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:59 smithi105 ceph-mon[65554]: pgmap v91: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 2.4 MiB/s wr, 213 op/s; 178/1647 objects degraded (10.808%) 2024-09-13T22:37:00.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:00.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:36:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:00.071 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:36:59 smithi105 podman[74285]: 2024-09-13 22:36:59.807865991 +0000 UTC m=+0.167952664 container create c70af9930f804bbc4fc8e21bfcb635a6abe60d428d6785e65308a3e3ad3d422b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731) 2024-09-13T22:37:00.332 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:37:00 smithi105 podman[74285]: 2024-09-13 22:37:00.236913501 +0000 UTC m=+0.597000181 container init c70af9930f804bbc4fc8e21bfcb635a6abe60d428d6785e65308a3e3ad3d422b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T22:37:00.332 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:37:00 smithi105 podman[74285]: 2024-09-13 22:37:00.248737257 +0000 UTC m=+0.608823934 container start c70af9930f804bbc4fc8e21bfcb635a6abe60d428d6785e65308a3e3ad3d422b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T22:37:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:59 smithi118 ceph-mon[51566]: pgmap v91: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 2.4 MiB/s wr, 213 op/s; 178/1647 objects degraded (10.808%) 2024-09-13T22:37:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:36:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:00.422 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:37:00 smithi105 bash[74285]: c70af9930f804bbc4fc8e21bfcb635a6abe60d428d6785e65308a3e3ad3d422b 2024-09-13T22:37:00.429 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:37:00 smithi105 systemd[1]: Started Ceph osd.1 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:37:00.567 INFO:teuthology.orchestra.run.smithi105.stdout: 5 167409 9.40 MB/sec execute 328 sec latency 35.403 ms 2024-09-13T22:37:01.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 167718 9.40 MB/sec execute 329 sec latency 64.978 ms 2024-09-13T22:37:01.622 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:01 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:01.622 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:01 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:01.622 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:01 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:37:01.622 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:01 smithi105 ceph-mon[65554]: pgmap v92: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 427 op/s; 184/1689 objects degraded (10.894%) 2024-09-13T22:37:01.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:01 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:01.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:01 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:01.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:01 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:37:01.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:01 smithi118 ceph-mon[51566]: pgmap v92: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 427 op/s; 184/1689 objects degraded (10.894%) 2024-09-13T22:37:02.390 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:37:02 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[74299]: 2024-09-13T22:37:02.378+0000 7fe7cd3e3640 -1 Falling back to public interface 2024-09-13T22:37:02.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 168103 9.40 MB/sec execute 330 sec latency 47.348 ms 2024-09-13T22:37:02.644 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:02 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 184/1689 objects degraded (10.894%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:02.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:02 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 184/1689 objects degraded (10.894%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:03.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 168501 9.41 MB/sec execute 331 sec latency 52.544 ms 2024-09-13T22:37:03.677 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:03 smithi105 ceph-mon[65554]: pgmap v93: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 395 op/s; 190/1707 objects degraded (11.131%) 2024-09-13T22:37:03.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:03 smithi118 ceph-mon[51566]: pgmap v93: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 395 op/s; 190/1707 objects degraded (11.131%) 2024-09-13T22:37:04.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 168820 9.41 MB/sec execute 332 sec latency 43.855 ms 2024-09-13T22:37:05.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 169137 9.41 MB/sec execute 333 sec latency 50.994 ms 2024-09-13T22:37:05.958 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:05 smithi105 ceph-mon[65554]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.8 MiB/s wr, 454 op/s; 191/1725 objects degraded (11.072%) 2024-09-13T22:37:06.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:05 smithi118 ceph-mon[51566]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.8 MiB/s wr, 454 op/s; 191/1725 objects degraded (11.072%) 2024-09-13T22:37:06.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 169481 9.40 MB/sec execute 334 sec latency 40.277 ms 2024-09-13T22:37:06.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:06 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 193/1734 objects degraded (11.130%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:07.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:06 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 193/1734 objects degraded (11.130%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:07.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 169709 9.39 MB/sec execute 335 sec latency 53.380 ms 2024-09-13T22:37:07.991 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:07 smithi105 ceph-mon[65554]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 417 op/s; 193/1734 objects degraded (11.130%) 2024-09-13T22:37:08.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:07 smithi118 ceph-mon[51566]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 417 op/s; 193/1734 objects degraded (11.130%) 2024-09-13T22:37:08.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 169952 9.39 MB/sec execute 336 sec latency 44.326 ms 2024-09-13T22:37:09.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 170235 9.39 MB/sec execute 337 sec latency 75.279 ms 2024-09-13T22:37:10.267 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:37:10 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[74299]: 2024-09-13T22:37:10.246+0000 7fe7cd3e3640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-13T22:37:10.267 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:10 smithi105 ceph-mon[65554]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 406 op/s; 193/1734 objects degraded (11.130%) 2024-09-13T22:37:10.267 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:10 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:10.267 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:10 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:10.267 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:10 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:10.268 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:10 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:10.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 170687 9.38 MB/sec execute 338 sec latency 42.518 ms 2024-09-13T22:37:10.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:10 smithi118 ceph-mon[51566]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 406 op/s; 193/1734 objects degraded (11.130%) 2024-09-13T22:37:10.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:10 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:10.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:10 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:10.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:10 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:10.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:10 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:11.282 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:37:11 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[74299]: 2024-09-13T22:37:11.214+0000 7fe7cd3e3640 -1 osd.1 85 log_to_monitors true 2024-09-13T22:37:11.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 171080 9.39 MB/sec execute 339 sec latency 35.955 ms 2024-09-13T22:37:12.331 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:12 smithi105 ceph-mon[65554]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 559 op/s; 182/1659 objects degraded (10.970%) 2024-09-13T22:37:12.331 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:12 smithi105 ceph-mon[65554]: from='osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T22:37:12.569 INFO:teuthology.orchestra.run.smithi105.stdout: 5 171406 9.39 MB/sec execute 340 sec latency 90.742 ms 2024-09-13T22:37:12.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:12 smithi118 ceph-mon[51566]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 559 op/s; 182/1659 objects degraded (10.970%) 2024-09-13T22:37:12.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:12 smithi118 ceph-mon[51566]: from='osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T22:37:12.835 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:37:12 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[74299]: 2024-09-13T22:37:12.727+0000 7fe7c1186640 -1 osd.1 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:37:13.339 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:13 smithi105 ceph-mon[65554]: from='osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T22:37:13.339 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:13 smithi105 ceph-mon[65554]: osdmap e88: 8 total, 7 up, 8 in 2024-09-13T22:37:13.339 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:13 smithi105 ceph-mon[65554]: from='osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:37:13.339 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:13 smithi105 ceph-mon[65554]: from='osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710]' entity='osd.1' 2024-09-13T22:37:13.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 172015 9.41 MB/sec execute 341 sec latency 27.492 ms 2024-09-13T22:37:13.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:13 smithi118 ceph-mon[51566]: from='osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T22:37:13.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:13 smithi118 ceph-mon[51566]: osdmap e88: 8 total, 7 up, 8 in 2024-09-13T22:37:13.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:13 smithi118 ceph-mon[51566]: from='osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:37:13.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:13 smithi118 ceph-mon[51566]: from='osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710]' entity='osd.1' 2024-09-13T22:37:14.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 172129 9.39 MB/sec execute 342 sec latency 876.662 ms 2024-09-13T22:37:14.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:14 smithi118 ceph-mon[51566]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 547 op/s; 182/1665 objects degraded (10.931%) 2024-09-13T22:37:14.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:14 smithi118 ceph-mon[51566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:37:14.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:14 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 182/1665 objects degraded (10.931%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:14.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:14 smithi118 ceph-mon[51566]: osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710] boot 2024-09-13T22:37:14.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:14 smithi118 ceph-mon[51566]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T22:37:14.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:37:14.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:14.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:14.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:14 smithi118 ceph-mon[51566]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T22:37:14.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:14 smithi105 ceph-mon[65554]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 547 op/s; 182/1665 objects degraded (10.931%) 2024-09-13T22:37:14.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:14 smithi105 ceph-mon[65554]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:37:14.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:14 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 182/1665 objects degraded (10.931%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:14.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:14 smithi105 ceph-mon[65554]: osd.1 [v2:172.21.15.105:6810/635040710,v1:172.21.15.105:6811/635040710] boot 2024-09-13T22:37:14.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:14 smithi105 ceph-mon[65554]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T22:37:14.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T22:37:14.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:14.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:14.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:14 smithi105 ceph-mon[65554]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T22:37:15.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 172374 9.39 MB/sec execute 343 sec latency 1393.504 ms 2024-09-13T22:37:16.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: pgmap v102: 97 pgs: 9 remapped+peering, 7 active+undersized, 10 active+undersized+degraded, 71 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 7.0 MiB/s wr, 720 op/s; 79/1704 objects degraded (4.636%); 0 B/s, 2 objects/s recovering 2024-09-13T22:37:16.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:16.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:16 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T22:37:16.242 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:37:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: pgmap v102: 97 pgs: 9 remapped+peering, 7 active+undersized, 10 active+undersized+degraded, 71 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 7.0 MiB/s wr, 720 op/s; 79/1704 objects degraded (4.636%); 0 B/s, 2 objects/s recovering 2024-09-13T22:37:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T22:37:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:16 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T22:37:16.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 172555 9.38 MB/sec execute 344 sec latency 565.899 ms 2024-09-13T22:37:16.934 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:37:16.934 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (5m) 7s ago 14m 17.6M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:37:16.934 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (101s) 7s ago 14m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (95s) 91s ago 13m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (4m) 7s ago 14m 45.2M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (8m) 7s ago 8m 38.0M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (8m) 91s ago 8m 14.3M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (4m) 7s ago 15m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (3m) 91s ago 13m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (3m) 7s ago 15m 48.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (2m) 91s ago 13m 36.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (7m) 7s ago 7m 266M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (7m) 91s ago 7m 81.1M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (4m) 7s ago 13m 7977k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (4m) 91s ago 13m 14.5M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (62s) 7s ago 12m 209M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (17s) 7s ago 12m 14.2M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:37:16.935 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (11m) 7s ago 11m 394M 1936M 17.2.0 e1d6a67b021e 830724a93f9f 2024-09-13T22:37:16.936 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (11m) 7s ago 11m 538M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:37:16.936 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (11m) 91s ago 10m 496M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:37:16.936 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (10m) 91s ago 10m 551M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:37:16.936 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (10m) 91s ago 10m 479M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:37:16.936 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (9m) 91s ago 9m 381M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:37:16.936 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (3m) 7s ago 13m 60.8M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:37:17.188 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:17 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T22:37:17.188 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:17 smithi105 ceph-mon[65554]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-13T22:37:17.188 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:17 smithi105 ceph-mon[65554]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T22:37:17.188 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:17 smithi105 ceph-mon[65554]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:17.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:17 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T22:37:17.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:17 smithi118 ceph-mon[51566]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-13T22:37:17.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:17 smithi118 ceph-mon[51566]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T22:37:17.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:17 smithi118 ceph-mon[51566]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:17.568 INFO:teuthology.orchestra.run.smithi105.stdout: 5 172692 9.35 MB/sec execute 345 sec latency 1046.148 ms 2024-09-13T22:37:17.587 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:37:17.587 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:37:17.587 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:37:17.587 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:37:17.587 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:37:17.587 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:37:17.587 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:37:17.587 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:37:17.587 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 6 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:37:17.588 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:37:18.011 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:37:18.011 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:37:18.011 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:37:18.011 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:37:18.012 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:37:18.012 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:37:18.012 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:37:18.012 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:37:18.012 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:37:18.012 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "8/23 daemons upgraded", 2024-09-13T22:37:18.012 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:37:18.012 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:37:18.012 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:37:18.204 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:18 smithi105 ceph-mon[65554]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:18.204 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:18 smithi105 ceph-mon[65554]: pgmap v105: 97 pgs: 1 peering, 11 remapped+peering, 4 active+undersized, 6 active+undersized+degraded, 75 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.4 MiB/s wr, 392 op/s; 33/1704 objects degraded (1.937%); 190 KiB/s, 7 objects/s recovering 2024-09-13T22:37:18.204 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:18 smithi105 ceph-mon[65554]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:18.204 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:18 smithi105 ceph-mon[65554]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T22:37:18.205 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:18 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/544250813' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:18 smithi118 ceph-mon[51566]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:18 smithi118 ceph-mon[51566]: pgmap v105: 97 pgs: 1 peering, 11 remapped+peering, 4 active+undersized, 6 active+undersized+degraded, 75 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.4 MiB/s wr, 392 op/s; 33/1704 objects degraded (1.937%); 190 KiB/s, 7 objects/s recovering 2024-09-13T22:37:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:18 smithi118 ceph-mon[51566]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:18 smithi118 ceph-mon[51566]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T22:37:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:18 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/544250813' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:18.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 172949 9.35 MB/sec execute 346 sec latency 994.345 ms 2024-09-13T22:37:18.640 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_WARN Degraded data redundancy: 33/1704 objects degraded (1.937%), 6 pgs degraded 2024-09-13T22:37:18.640 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 33/1704 objects degraded (1.937%), 6 pgs degraded 2024-09-13T22:37:18.640 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-13T22:37:18.641 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-13T22:37:18.641 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-13T22:37:18.641 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-13T22:37:18.641 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-13T22:37:18.641 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-13T22:37:19.215 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:19 smithi105 ceph-mon[65554]: from='client.44185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:19.215 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:19 smithi105 ceph-mon[65554]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T22:37:19.215 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:19 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/195155543' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:37:19.215 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:19 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 33/1704 objects degraded (1.937%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:19 smithi118 ceph-mon[51566]: from='client.44185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:19 smithi118 ceph-mon[51566]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T22:37:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:19 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/195155543' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:37:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:19 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 33/1704 objects degraded (1.937%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:19.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 173358 9.35 MB/sec execute 347 sec latency 816.600 ms 2024-09-13T22:37:20.235 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:20 smithi105 ceph-mon[65554]: pgmap v108: 97 pgs: 2 peering, 9 remapped+peering, 86 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 1.6 MiB/s wr, 137 op/s; 2.2 MiB/s, 64 keys/s, 17 objects/s recovering 2024-09-13T22:37:20.235 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:20 smithi105 ceph-mon[65554]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1704 objects degraded (1.937%), 6 pgs degraded) 2024-09-13T22:37:20.235 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:20 smithi105 ceph-mon[65554]: Cluster is now healthy 2024-09-13T22:37:20.235 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:20 smithi105 ceph-mon[65554]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T22:37:20.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:20 smithi118 ceph-mon[51566]: pgmap v108: 97 pgs: 2 peering, 9 remapped+peering, 86 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 1.6 MiB/s wr, 137 op/s; 2.2 MiB/s, 64 keys/s, 17 objects/s recovering 2024-09-13T22:37:20.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:20 smithi118 ceph-mon[51566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1704 objects degraded (1.937%), 6 pgs degraded) 2024-09-13T22:37:20.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:20 smithi118 ceph-mon[51566]: Cluster is now healthy 2024-09-13T22:37:20.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:20 smithi118 ceph-mon[51566]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T22:37:20.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 173692 9.35 MB/sec execute 348 sec latency 71.537 ms 2024-09-13T22:37:21.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 174032 9.35 MB/sec execute 349 sec latency 41.230 ms 2024-09-13T22:37:22.263 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:22 smithi105 ceph-mon[65554]: pgmap v110: 97 pgs: 2 peering, 1 remapped+peering, 94 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.2 MiB/s wr, 269 op/s; 3.2 MiB/s, 53 keys/s, 45 objects/s recovering 2024-09-13T22:37:22.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:22 smithi118 ceph-mon[51566]: pgmap v110: 97 pgs: 2 peering, 1 remapped+peering, 94 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.2 MiB/s wr, 269 op/s; 3.2 MiB/s, 53 keys/s, 45 objects/s recovering 2024-09-13T22:37:22.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 174443 9.36 MB/sec execute 350 sec latency 41.398 ms 2024-09-13T22:37:23.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 174739 9.35 MB/sec execute 351 sec latency 38.286 ms 2024-09-13T22:37:24.312 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:24 smithi105 ceph-mon[65554]: pgmap v111: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 399 op/s; 2.5 MiB/s, 43 keys/s, 40 objects/s recovering 2024-09-13T22:37:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:24 smithi118 ceph-mon[51566]: pgmap v111: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 399 op/s; 2.5 MiB/s, 43 keys/s, 40 objects/s recovering 2024-09-13T22:37:24.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 175069 9.36 MB/sec execute 352 sec latency 55.359 ms 2024-09-13T22:37:25.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 175431 9.35 MB/sec execute 353 sec latency 34.460 ms 2024-09-13T22:37:26.079 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:26 smithi105 ceph-mon[65554]: pgmap v112: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.6 MiB/s wr, 388 op/s; 1.9 MiB/s, 33 keys/s, 30 objects/s recovering 2024-09-13T22:37:26.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:26 smithi118 ceph-mon[51566]: pgmap v112: 97 pgs: 97 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.6 MiB/s wr, 388 op/s; 1.9 MiB/s, 33 keys/s, 30 objects/s recovering 2024-09-13T22:37:26.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 175796 9.35 MB/sec execute 354 sec latency 61.924 ms 2024-09-13T22:37:27.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 176074 9.34 MB/sec execute 355 sec latency 45.010 ms 2024-09-13T22:37:28.129 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:28 smithi105 ceph-mon[65554]: pgmap v113: 97 pgs: 97 active+clean; 360 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 499 op/s; 803 KiB/s, 21 objects/s recovering 2024-09-13T22:37:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:28 smithi118 ceph-mon[51566]: pgmap v113: 97 pgs: 97 active+clean; 360 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 499 op/s; 803 KiB/s, 21 objects/s recovering 2024-09-13T22:37:28.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 176308 9.34 MB/sec execute 356 sec latency 58.119 ms 2024-09-13T22:37:29.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 176642 9.34 MB/sec execute 357 sec latency 65.085 ms 2024-09-13T22:37:29.929 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:29 smithi105 ceph-mon[65554]: pgmap v114: 97 pgs: 97 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 448 op/s; 708 KiB/s, 18 objects/s recovering 2024-09-13T22:37:29.929 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:29.929 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:29.929 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:29 smithi118 ceph-mon[51566]: pgmap v114: 97 pgs: 97 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 448 op/s; 708 KiB/s, 18 objects/s recovering 2024-09-13T22:37:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:30.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 176929 9.33 MB/sec execute 358 sec latency 63.731 ms 2024-09-13T22:37:30.946 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T22:37:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T22:37:31.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 177222 9.33 MB/sec execute 359 sec latency 49.161 ms 2024-09-13T22:37:31.967 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:31 smithi105 ceph-mon[65554]: pgmap v115: 97 pgs: 97 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 544 op/s; 599 KiB/s, 16 objects/s recovering 2024-09-13T22:37:31.967 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:31 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T22:37:31.967 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:31 smithi105 ceph-mon[65554]: Upgrade: osd.2 is safe to restart 2024-09-13T22:37:32.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:31 smithi118 ceph-mon[51566]: pgmap v115: 97 pgs: 97 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 544 op/s; 599 KiB/s, 16 objects/s recovering 2024-09-13T22:37:32.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:31 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T22:37:32.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:31 smithi118 ceph-mon[51566]: Upgrade: osd.2 is safe to restart 2024-09-13T22:37:32.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 177495 9.33 MB/sec execute 360 sec latency 64.542 ms 2024-09-13T22:37:33.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 177878 9.33 MB/sec execute 361 sec latency 54.341 ms 2024-09-13T22:37:34.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:33 smithi105 ceph-mon[65554]: pgmap v116: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 486 op/s; 0 B/s, 3 objects/s recovering 2024-09-13T22:37:34.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:33 smithi105 ceph-mon[65554]: Upgrade: Updating osd.2 2024-09-13T22:37:34.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:33 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:34.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:33 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T22:37:34.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:33 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:37:34.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:33 smithi105 ceph-mon[65554]: Deploying daemon osd.2 on smithi105 2024-09-13T22:37:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:33 smithi118 ceph-mon[51566]: pgmap v116: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 486 op/s; 0 B/s, 3 objects/s recovering 2024-09-13T22:37:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:33 smithi118 ceph-mon[51566]: Upgrade: Updating osd.2 2024-09-13T22:37:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:33 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:33 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T22:37:34.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:33 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:37:34.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:33 smithi118 ceph-mon[51566]: Deploying daemon osd.2 on smithi105 2024-09-13T22:37:34.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 178189 9.33 MB/sec execute 362 sec latency 38.753 ms 2024-09-13T22:37:35.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 178513 9.33 MB/sec execute 363 sec latency 37.333 ms 2024-09-13T22:37:36.059 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:36 smithi105 ceph-mon[65554]: pgmap v117: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 542 op/s 2024-09-13T22:37:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:36 smithi118 ceph-mon[51566]: pgmap v117: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 542 op/s 2024-09-13T22:37:36.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 178748 9.32 MB/sec execute 364 sec latency 46.979 ms 2024-09-13T22:37:37.335 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:37 smithi105 systemd[1]: Stopping Ceph osd.2 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:37:37.575 INFO:teuthology.orchestra.run.smithi105.stdout: 5 179060 9.31 MB/sec execute 365 sec latency 57.528 ms 2024-09-13T22:37:37.841 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:37 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[40957]: 2024-09-13T22:37:37.696+0000 7f756bcd7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:37:37.841 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:37 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[40957]: 2024-09-13T22:37:37.696+0000 7f756bcd7700 -1 osd.2 95 *** Got signal Terminated *** 2024-09-13T22:37:37.841 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:37 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[40957]: 2024-09-13T22:37:37.696+0000 7f756bcd7700 -1 osd.2 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:37:38.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:38 smithi105 ceph-mon[65554]: pgmap v118: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 503 op/s 2024-09-13T22:37:38.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:38 smithi105 ceph-mon[65554]: osd.2 marked itself down and dead 2024-09-13T22:37:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:38 smithi118 ceph-mon[51566]: pgmap v118: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.7 MiB/s wr, 503 op/s 2024-09-13T22:37:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:38 smithi118 ceph-mon[51566]: osd.2 marked itself down and dead 2024-09-13T22:37:38.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 179214 9.30 MB/sec execute 366 sec latency 565.258 ms 2024-09-13T22:37:38.917 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:38 smithi105 podman[76455]: 2024-09-13 22:37:38.673703967 +0000 UTC m=+1.149183540 container died 830724a93f9f41f09858a36e26478ba715b9d12fdc7330155d070c3788efad94 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, version=8, ceph=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, vendor=Red Hat, Inc., RELEASE=HEAD, architecture=x86_64, io.openshift.expose-services=, release=754, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:37:39.169 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:39 smithi105 ceph-mon[65554]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:37:39.169 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:39 smithi105 ceph-mon[65554]: osdmap e96: 8 total, 7 up, 8 in 2024-09-13T22:37:39.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:39 smithi118 ceph-mon[51566]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:37:39.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:39 smithi118 ceph-mon[51566]: osdmap e96: 8 total, 7 up, 8 in 2024-09-13T22:37:39.422 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:39 smithi105 podman[76455]: 2024-09-13 22:37:39.233224657 +0000 UTC m=+1.708704234 container cleanup 830724a93f9f41f09858a36e26478ba715b9d12fdc7330155d070c3788efad94 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, version=8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, architecture=x86_64) 2024-09-13T22:37:39.422 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:39 smithi105 bash[76455]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2 2024-09-13T22:37:39.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 179463 9.28 MB/sec execute 367 sec latency 1049.858 ms 2024-09-13T22:37:39.673 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:39 smithi105 podman[76473]: 2024-09-13 22:37:39.571460444 +0000 UTC m=+0.896576418 container remove 830724a93f9f41f09858a36e26478ba715b9d12fdc7330155d070c3788efad94 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , ceph=True, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T22:37:40.196 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:40 smithi105 ceph-mon[65554]: pgmap v120: 97 pgs: 5 stale+active+clean, 92 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 520 op/s 2024-09-13T22:37:40.197 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:40 smithi105 ceph-mon[65554]: osdmap e97: 8 total, 7 up, 8 in 2024-09-13T22:37:40.197 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:40 smithi105 podman[76561]: 2024-09-13 22:37:40.010836759 +0000 UTC m=+0.021120615 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:37:40.197 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:40 smithi105 podman[76561]: 2024-09-13 22:37:40.130051996 +0000 UTC m=+0.140335847 container create 4364c749f7eb47fe254b8077f533a64da5b587b3f0eac00eb59f87b79640e78e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:37:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:40 smithi118 ceph-mon[51566]: pgmap v120: 97 pgs: 5 stale+active+clean, 92 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 520 op/s 2024-09-13T22:37:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:40 smithi118 ceph-mon[51566]: osdmap e97: 8 total, 7 up, 8 in 2024-09-13T22:37:40.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 179870 9.29 MB/sec execute 368 sec latency 51.076 ms 2024-09-13T22:37:40.698 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:40 smithi105 podman[76561]: 2024-09-13 22:37:40.458618761 +0000 UTC m=+0.468902631 container init 4364c749f7eb47fe254b8077f533a64da5b587b3f0eac00eb59f87b79640e78e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T22:37:40.698 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:40 smithi105 podman[76561]: 2024-09-13 22:37:40.464301719 +0000 UTC m=+0.474585578 container start 4364c749f7eb47fe254b8077f533a64da5b587b3f0eac00eb59f87b79640e78e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-13T22:37:40.698 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:40 smithi105 podman[76561]: 2024-09-13 22:37:40.635587042 +0000 UTC m=+0.645870899 container attach 4364c749f7eb47fe254b8077f533a64da5b587b3f0eac00eb59f87b79640e78e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be) 2024-09-13T22:37:40.950 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:40 smithi105 podman[76561]: 2024-09-13 22:37:40.791650062 +0000 UTC m=+0.801933923 container died 4364c749f7eb47fe254b8077f533a64da5b587b3f0eac00eb59f87b79640e78e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T22:37:41.205 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:41 smithi105 ceph-mon[65554]: Health check failed: Degraded data redundancy: 97/1635 objects degraded (5.933%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:41.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:41 smithi118 ceph-mon[51566]: Health check failed: Degraded data redundancy: 97/1635 objects degraded (5.933%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:41.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 180456 9.31 MB/sec execute 369 sec latency 61.465 ms 2024-09-13T22:37:41.980 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:41 smithi105 podman[76583]: 2024-09-13 22:37:41.824763651 +0000 UTC m=+1.016239512 container remove 4364c749f7eb47fe254b8077f533a64da5b587b3f0eac00eb59f87b79640e78e (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-13T22:37:41.981 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:41 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T22:37:41.981 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:41 smithi105 systemd[1]: Stopped Ceph osd.2 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:37:41.981 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:41 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 50ms CPU time. 2024-09-13T22:37:42.240 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:42 smithi105 ceph-mon[65554]: pgmap v122: 97 pgs: 3 active+undersized, 2 stale+active+clean, 15 active+undersized+degraded, 77 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 487 op/s; 97/1635 objects degraded (5.933%) 2024-09-13T22:37:42.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:42 smithi118 ceph-mon[51566]: pgmap v122: 97 pgs: 3 active+undersized, 2 stale+active+clean, 15 active+undersized+degraded, 77 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 487 op/s; 97/1635 objects degraded (5.933%) 2024-09-13T22:37:42.492 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:42 smithi105 systemd[1]: Starting Ceph osd.2 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:37:42.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 180789 9.31 MB/sec execute 370 sec latency 42.471 ms 2024-09-13T22:37:42.745 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:42 smithi105 podman[76712]: 2024-09-13 22:37:42.410626326 +0000 UTC m=+0.023467838 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:37:42.746 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:42 smithi105 podman[76712]: 2024-09-13 22:37:42.593191958 +0000 UTC m=+0.206033460 container create c0ffc61008327b0366be7d593f887ec979326f490a87d3f8399ffa61c7c4c260 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate, GIT_CLEAN=True, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-13T22:37:43.249 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:43 smithi105 podman[76712]: 2024-09-13 22:37:43.022048188 +0000 UTC m=+0.634889694 container init c0ffc61008327b0366be7d593f887ec979326f490a87d3f8399ffa61c7c4c260 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be) 2024-09-13T22:37:43.249 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:43 smithi105 podman[76712]: 2024-09-13 22:37:43.03757894 +0000 UTC m=+0.650420443 container start c0ffc61008327b0366be7d593f887ec979326f490a87d3f8399ffa61c7c4c260 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-13T22:37:43.249 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:43 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate[76726]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:37:43.503 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:43 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate[76726]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:37:43.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 181040 9.31 MB/sec execute 371 sec latency 132.373 ms 2024-09-13T22:37:44.270 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-mon[65554]: pgmap v123: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.1 MiB/s wr, 309 op/s; 155/1686 objects degraded (9.193%) 2024-09-13T22:37:44.270 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:44.270 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:44.271 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 bash[76712]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:37:44.271 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 bash[76712]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:37:44.271 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 podman[76712]: 2024-09-13 22:37:44.125348923 +0000 UTC m=+1.738190432 container attach c0ffc61008327b0366be7d593f887ec979326f490a87d3f8399ffa61c7c4c260 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:37:44.271 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate[76726]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T22:37:44.271 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 bash[76712]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T22:37:44.271 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate[76726]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T22:37:44.271 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 bash[76712]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T22:37:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:44 smithi118 ceph-mon[51566]: pgmap v123: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.1 MiB/s wr, 309 op/s; 155/1686 objects degraded (9.193%) 2024-09-13T22:37:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:44.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 181432 9.31 MB/sec execute 372 sec latency 56.612 ms 2024-09-13T22:37:44.779 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate[76726]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T22:37:44.779 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 bash[76712]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T22:37:44.779 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate[76726]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T22:37:44.780 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 bash[76712]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T22:37:44.780 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate[76726]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T22:37:44.780 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 bash[76712]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T22:37:44.780 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate[76726]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T22:37:44.780 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 bash[76712]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T22:37:44.780 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate[76726]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T22:37:44.780 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 bash[76712]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T22:37:44.780 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:44 smithi105 podman[76712]: 2024-09-13 22:37:44.706552497 +0000 UTC m=+2.319394004 container died c0ffc61008327b0366be7d593f887ec979326f490a87d3f8399ffa61c7c4c260 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T22:37:45.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 182111 9.35 MB/sec execute 373 sec latency 28.405 ms 2024-09-13T22:37:46.034 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:45 smithi105 podman[76876]: 2024-09-13 22:37:45.873519942 +0000 UTC m=+1.160687247 container remove c0ffc61008327b0366be7d593f887ec979326f490a87d3f8399ffa61c7c4c260 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:37:46.288 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:46 smithi105 ceph-mon[65554]: pgmap v124: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 596 op/s; 154/1740 objects degraded (8.851%) 2024-09-13T22:37:46.288 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:46 smithi105 podman[76932]: 2024-09-13 22:37:46.035536656 +0000 UTC m=+0.027722640 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:37:46.288 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:46 smithi105 podman[76932]: 2024-09-13 22:37:46.258676424 +0000 UTC m=+0.250862398 container create 71d2466dd952bd2c2357bd204776e5ac3503b3e557889aa65ebb9e8259e8b696 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T22:37:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:46 smithi118 ceph-mon[51566]: pgmap v124: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 596 op/s; 154/1740 objects degraded (8.851%) 2024-09-13T22:37:46.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 182876 9.36 MB/sec execute 374 sec latency 40.469 ms 2024-09-13T22:37:46.790 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:46 smithi105 podman[76932]: 2024-09-13 22:37:46.668529884 +0000 UTC m=+0.660715884 container init 71d2466dd952bd2c2357bd204776e5ac3503b3e557889aa65ebb9e8259e8b696 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-13T22:37:46.790 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:46 smithi105 podman[76932]: 2024-09-13 22:37:46.674764847 +0000 UTC m=+0.666950822 container start 71d2466dd952bd2c2357bd204776e5ac3503b3e557889aa65ebb9e8259e8b696 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True) 2024-09-13T22:37:46.790 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:46 smithi105 ceph-osd[76949]: -- 172.21.15.105:0/2032476166 <== mon.0 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55be9f5181e0 con 0x55be9e705c00 2024-09-13T22:37:47.041 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:46 smithi105 bash[76932]: 71d2466dd952bd2c2357bd204776e5ac3503b3e557889aa65ebb9e8259e8b696 2024-09-13T22:37:47.042 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:46 smithi105 systemd[1]: Started Ceph osd.2 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:37:47.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 183507 9.37 MB/sec execute 375 sec latency 26.371 ms 2024-09-13T22:37:48.049 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:47 smithi105 ceph-mon[65554]: pgmap v125: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 458 op/s; 154/1734 objects degraded (8.881%) 2024-09-13T22:37:48.049 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:48.049 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:48.049 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:37:48.049 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:47 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 154/1734 objects degraded (8.881%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:47 smithi118 ceph-mon[51566]: pgmap v125: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 458 op/s; 154/1734 objects degraded (8.881%) 2024-09-13T22:37:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:37:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:47 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 154/1734 objects degraded (8.881%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:48.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 184156 9.40 MB/sec execute 376 sec latency 42.829 ms 2024-09-13T22:37:48.800 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:48 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[76945]: 2024-09-13T22:37:48.637+0000 7f01c5ba3640 -1 Falling back to public interface 2024-09-13T22:37:49.232 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:37:49.576 INFO:teuthology.orchestra.run.smithi105.stdout: 5 184561 9.41 MB/sec execute 377 sec latency 45.201 ms 2024-09-13T22:37:50.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:49 smithi105 ceph-mon[65554]: pgmap v126: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 416 op/s; 159/1749 objects degraded (9.091%) 2024-09-13T22:37:50.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:49 smithi105 ceph-mon[65554]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:50.336 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:37:50.336 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (5m) 41s ago 14m 17.6M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:37:50.336 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (2m) 41s ago 14m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:37:50.336 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (2m) 2m ago 14m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:37:50.336 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (5m) 41s ago 14m 45.2M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:37:50.336 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (8m) 41s ago 8m 38.0M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:37:50.336 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (8m) 2m ago 8m 14.3M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:37:50.336 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (4m) 41s ago 16m 503M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (3m) 2m ago 14m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (3m) 41s ago 16m 48.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (3m) 2m ago 14m 36.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (8m) 41s ago 8m 266M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (8m) 2m ago 8m 81.1M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (5m) 41s ago 14m 7977k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (4m) 2m ago 13m 14.5M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (95s) 41s ago 13m 209M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (50s) 41s ago 12m 14.2M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 starting - - - 1936M 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (11m) 41s ago 11m 538M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (11m) 2m ago 11m 496M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:37:50.337 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (11m) 2m ago 11m 551M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:37:50.338 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (10m) 2m ago 10m 479M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:37:50.338 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (10m) 2m ago 10m 381M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:37:50.338 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (4m) 41s ago 13m 60.8M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:37:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:49 smithi118 ceph-mon[51566]: pgmap v126: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 416 op/s; 159/1749 objects degraded (9.091%) 2024-09-13T22:37:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:49 smithi118 ceph-mon[51566]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:50.577 INFO:teuthology.orchestra.run.smithi105.stdout: 5 184989 9.41 MB/sec execute 378 sec latency 62.252 ms 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:37:50.947 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:37:50.948 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:37:50.948 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:37:50.948 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:37:50.948 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:37:50.948 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T22:37:50.948 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 6 2024-09-13T22:37:50.948 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:37:50.948 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:37:51.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:50 smithi105 ceph-mon[65554]: from='client.44197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:51.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:50 smithi105 ceph-mon[65554]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:51.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:50 smithi118 ceph-mon[51566]: from='client.44197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:51.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:50 smithi118 ceph-mon[51566]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:51.408 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:37:51.408 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:37:51.408 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:37:51.408 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:37:51.408 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:37:51.408 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:37:51.408 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:37:51.408 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:37:51.409 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:37:51.409 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "8/23 daemons upgraded", 2024-09-13T22:37:51.409 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:37:51.409 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:37:51.409 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:37:51.577 INFO:teuthology.orchestra.run.smithi105.stdout: 5 185248 9.41 MB/sec execute 379 sec latency 53.347 ms 2024-09-13T22:37:51.926 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 172/1770 objects degraded (9.718%), 20 pgs degraded 2024-09-13T22:37:51.926 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T22:37:51.926 INFO:teuthology.orchestra.run.smithi105.stdout: osd.2 (root=default,host=smithi105) is down 2024-09-13T22:37:51.926 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 172/1770 objects degraded (9.718%), 20 pgs degraded 2024-09-13T22:37:51.926 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-13T22:37:51.927 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-13T22:37:51.928 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-13T22:37:51.928 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-13T22:37:51.928 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-13T22:37:51.928 INFO:teuthology.orchestra.run.smithi105.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-09-13T22:37:51.928 INFO:teuthology.orchestra.run.smithi105.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-13T22:37:52.100 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:51 smithi105 ceph-mon[65554]: pgmap v127: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 361 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 698 op/s; 172/1770 objects degraded (9.718%) 2024-09-13T22:37:52.100 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:51 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/527916521' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:52.100 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:51 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/4119317832' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:37:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:51 smithi118 ceph-mon[51566]: pgmap v127: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 361 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 698 op/s; 172/1770 objects degraded (9.718%) 2024-09-13T22:37:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:51 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/527916521' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:37:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:51 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/4119317832' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:37:52.577 INFO:teuthology.orchestra.run.smithi105.stdout: 5 185515 9.41 MB/sec execute 380 sec latency 60.846 ms 2024-09-13T22:37:53.155 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:52 smithi105 ceph-mon[65554]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:53.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:52 smithi118 ceph-mon[51566]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:37:53.577 INFO:teuthology.orchestra.run.smithi105.stdout: 5 185765 9.41 MB/sec execute 381 sec latency 53.401 ms 2024-09-13T22:37:54.183 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:53 smithi105 ceph-mon[65554]: pgmap v128: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.6 MiB/s wr, 647 op/s; 165/1716 objects degraded (9.615%) 2024-09-13T22:37:54.184 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:53 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 165/1716 objects degraded (9.615%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:53 smithi118 ceph-mon[51566]: pgmap v128: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.6 MiB/s wr, 647 op/s; 165/1716 objects degraded (9.615%) 2024-09-13T22:37:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:53 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 165/1716 objects degraded (9.615%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:37:54.577 INFO:teuthology.orchestra.run.smithi105.stdout: 5 186065 9.40 MB/sec execute 382 sec latency 70.370 ms 2024-09-13T22:37:55.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 186357 9.39 MB/sec execute 383 sec latency 59.387 ms 2024-09-13T22:37:55.715 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:55 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[76945]: 2024-09-13T22:37:55.706+0000 7f01c5ba3640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-13T22:37:55.966 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:55 smithi105 ceph-mon[65554]: pgmap v129: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 774 op/s; 157/1674 objects degraded (9.379%) 2024-09-13T22:37:56.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:55 smithi118 ceph-mon[51566]: pgmap v129: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 774 op/s; 157/1674 objects degraded (9.379%) 2024-09-13T22:37:56.493 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:56 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[76945]: 2024-09-13T22:37:56.484+0000 7f01c5ba3640 -1 osd.2 95 log_to_monitors true 2024-09-13T22:37:56.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 186582 9.37 MB/sec execute 384 sec latency 50.737 ms 2024-09-13T22:37:57.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:56 smithi105 ceph-mon[65554]: from='osd.2 [v2:172.21.15.105:6818/1568788150,v1:172.21.15.105:6819/1568788150]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T22:37:57.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:56 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:57.002 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:56 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:57.257 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:37:57 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[76945]: 2024-09-13T22:37:57.032+0000 7f01bd94e640 -1 osd.2 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:37:57.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:56 smithi118 ceph-mon[51566]: from='osd.2 [v2:172.21.15.105:6818/1568788150,v1:172.21.15.105:6819/1568788150]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T22:37:57.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:56 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:57.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:56 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:57.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 186844 9.37 MB/sec execute 385 sec latency 44.993 ms 2024-09-13T22:37:58.030 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:57 smithi105 ceph-mon[65554]: pgmap v130: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 580 op/s; 152/1647 objects degraded (9.229%) 2024-09-13T22:37:58.030 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:57 smithi105 ceph-mon[65554]: from='osd.2 [v2:172.21.15.105:6818/1568788150,v1:172.21.15.105:6819/1568788150]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T22:37:58.030 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:57 smithi105 ceph-mon[65554]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T22:37:58.030 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:57 smithi105 ceph-mon[65554]: from='osd.2 [v2:172.21.15.105:6818/1568788150,v1:172.21.15.105:6819/1568788150]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:37:58.030 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:58.031 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:57 smithi118 ceph-mon[51566]: pgmap v130: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 580 op/s; 152/1647 objects degraded (9.229%) 2024-09-13T22:37:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:57 smithi118 ceph-mon[51566]: from='osd.2 [v2:172.21.15.105:6818/1568788150,v1:172.21.15.105:6819/1568788150]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T22:37:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:57 smithi118 ceph-mon[51566]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T22:37:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:57 smithi118 ceph-mon[51566]: from='osd.2 [v2:172.21.15.105:6818/1568788150,v1:172.21.15.105:6819/1568788150]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:37:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:58.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 186928 9.35 MB/sec execute 386 sec latency 648.734 ms 2024-09-13T22:37:59.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:59 smithi105 ceph-mon[65554]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:37:59.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:59 smithi105 ceph-mon[65554]: osd.2 [v2:172.21.15.105:6818/1568788150,v1:172.21.15.105:6819/1568788150] boot 2024-09-13T22:37:59.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:59 smithi105 ceph-mon[65554]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T22:37:59.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:37:59.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:59.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:59.279 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:37:59 smithi105 ceph-mon[65554]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T22:37:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:59 smithi118 ceph-mon[51566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:37:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:59 smithi118 ceph-mon[51566]: osd.2 [v2:172.21.15.105:6818/1568788150,v1:172.21.15.105:6819/1568788150] boot 2024-09-13T22:37:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:59 smithi118 ceph-mon[51566]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T22:37:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T22:37:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:37:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:37:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:37:59 smithi118 ceph-mon[51566]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T22:37:59.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 186928 9.33 MB/sec execute 387 sec latency 1648.759 ms 2024-09-13T22:38:00.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:00 smithi105 ceph-mon[65554]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.6 MiB/s wr, 321 op/s; 149/1647 objects degraded (9.047%) 2024-09-13T22:38:00.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:00 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 149/1647 objects degraded (9.047%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:00.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:00 smithi105 ceph-mon[65554]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T22:38:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:00 smithi118 ceph-mon[51566]: pgmap v133: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.6 MiB/s wr, 321 op/s; 149/1647 objects degraded (9.047%) 2024-09-13T22:38:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:00 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 149/1647 objects degraded (9.047%), 20 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:00 smithi118 ceph-mon[51566]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T22:38:00.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 187127 9.32 MB/sec execute 388 sec latency 2270.795 ms 2024-09-13T22:38:01.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 187629 9.33 MB/sec execute 389 sec latency 51.157 ms 2024-09-13T22:38:02.278 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:02 smithi105 ceph-mon[65554]: pgmap v136: 97 pgs: 8 remapped+peering, 3 active+undersized, 7 active+undersized+degraded, 79 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.7 MiB/s wr, 299 op/s; 58/1650 objects degraded (3.515%) 2024-09-13T22:38:02.278 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:02 smithi105 ceph-mon[65554]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T22:38:02.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:02 smithi118 ceph-mon[51566]: pgmap v136: 97 pgs: 8 remapped+peering, 3 active+undersized, 7 active+undersized+degraded, 79 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.7 MiB/s wr, 299 op/s; 58/1650 objects degraded (3.515%) 2024-09-13T22:38:02.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:02 smithi118 ceph-mon[51566]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T22:38:02.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 187955 9.33 MB/sec execute 390 sec latency 491.551 ms 2024-09-13T22:38:03.036 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T22:38:03.036 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:03.036 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:03.036 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:38:03.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:38:03.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:03.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:03.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:38:03.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:03.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:03.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:03.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T22:38:03.037 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:03 smithi105 ceph-mon[65554]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T22:38:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T22:38:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:38:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:38:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:38:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T22:38:03.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:03 smithi118 ceph-mon[51566]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T22:38:03.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 188241 9.32 MB/sec execute 391 sec latency 950.554 ms 2024-09-13T22:38:04.568 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:04 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T22:38:04.568 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:04 smithi105 ceph-mon[65554]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-13T22:38:04.568 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:04 smithi105 ceph-mon[65554]: pgmap v139: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 7 remapped+peering, 86 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 506 op/s; 69/1632 objects degraded (4.228%); 982 KiB/s, 7 objects/s recovering 2024-09-13T22:38:04.568 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:04 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 69/1632 objects degraded (4.228%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:04.568 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:04 smithi105 ceph-mon[65554]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T22:38:04.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 188493 9.32 MB/sec execute 392 sec latency 1022.405 ms 2024-09-13T22:38:04.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:04 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T22:38:04.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:04 smithi118 ceph-mon[51566]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-13T22:38:04.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:04 smithi118 ceph-mon[51566]: pgmap v139: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 7 remapped+peering, 86 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 506 op/s; 69/1632 objects degraded (4.228%); 982 KiB/s, 7 objects/s recovering 2024-09-13T22:38:04.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:04 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 69/1632 objects degraded (4.228%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:04.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:04 smithi118 ceph-mon[51566]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T22:38:05.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 188757 9.32 MB/sec execute 393 sec latency 72.848 ms 2024-09-13T22:38:06.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 189081 9.33 MB/sec execute 394 sec latency 50.268 ms 2024-09-13T22:38:06.613 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:06 smithi105 ceph-mon[65554]: pgmap v142: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 93 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s; 69/1647 objects degraded (4.189%); 3.9 MiB/s, 38 objects/s recovering 2024-09-13T22:38:06.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:06 smithi118 ceph-mon[51566]: pgmap v142: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 93 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s; 69/1647 objects degraded (4.189%); 3.9 MiB/s, 38 objects/s recovering 2024-09-13T22:38:07.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 189376 9.32 MB/sec execute 395 sec latency 54.415 ms 2024-09-13T22:38:08.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:08 smithi105 ceph-mon[65554]: pgmap v143: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 94 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.9 KiB/s rd, 3.8 MiB/s wr, 327 op/s; 33/1650 objects degraded (2.000%); 2.7 MiB/s, 32 objects/s recovering 2024-09-13T22:38:08.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 189814 9.32 MB/sec execute 396 sec latency 32.529 ms 2024-09-13T22:38:08.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:08 smithi118 ceph-mon[51566]: pgmap v143: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 94 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.9 KiB/s rd, 3.8 MiB/s wr, 327 op/s; 33/1650 objects degraded (2.000%); 2.7 MiB/s, 32 objects/s recovering 2024-09-13T22:38:09.494 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:09 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 33/1650 objects degraded (2.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T22:38:09.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 190030 9.30 MB/sec execute 397 sec latency 86.917 ms 2024-09-13T22:38:09.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:09 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 33/1650 objects degraded (2.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T22:38:10.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:10 smithi105 ceph-mon[65554]: pgmap v144: 97 pgs: 97 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.8 MiB/s wr, 231 op/s; 1.8 MiB/s, 32 objects/s recovering 2024-09-13T22:38:10.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:10 smithi105 ceph-mon[65554]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1650 objects degraded (2.000%), 1 pg degraded) 2024-09-13T22:38:10.548 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:10 smithi105 ceph-mon[65554]: Cluster is now healthy 2024-09-13T22:38:10.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 190355 9.30 MB/sec execute 398 sec latency 45.263 ms 2024-09-13T22:38:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:10 smithi118 ceph-mon[51566]: pgmap v144: 97 pgs: 97 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.8 MiB/s wr, 231 op/s; 1.8 MiB/s, 32 objects/s recovering 2024-09-13T22:38:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:10 smithi118 ceph-mon[51566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1650 objects degraded (2.000%), 1 pg degraded) 2024-09-13T22:38:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:10 smithi118 ceph-mon[51566]: Cluster is now healthy 2024-09-13T22:38:11.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 190852 9.32 MB/sec execute 399 sec latency 50.115 ms 2024-09-13T22:38:12.558 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:12 smithi105 ceph-mon[65554]: pgmap v145: 97 pgs: 97 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.8 MiB/s wr, 457 op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-13T22:38:12.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 191414 9.33 MB/sec execute 400 sec latency 29.322 ms 2024-09-13T22:38:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:12 smithi118 ceph-mon[51566]: pgmap v145: 97 pgs: 97 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.8 MiB/s wr, 457 op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-13T22:38:13.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 191950 9.34 MB/sec execute 401 sec latency 30.960 ms 2024-09-13T22:38:14.453 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:14 smithi105 ceph-mon[65554]: pgmap v146: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 477 op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-13T22:38:14.453 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:14.453 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:38:14.453 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:14.578 INFO:teuthology.orchestra.run.smithi105.stdout: 5 192402 9.35 MB/sec execute 402 sec latency 26.814 ms 2024-09-13T22:38:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:14 smithi118 ceph-mon[51566]: pgmap v146: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 477 op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-13T22:38:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:38:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:15.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 192814 9.36 MB/sec execute 403 sec latency 64.080 ms 2024-09-13T22:38:16.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 193198 9.36 MB/sec execute 404 sec latency 40.855 ms 2024-09-13T22:38:16.597 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:16 smithi105 ceph-mon[65554]: pgmap v147: 97 pgs: 97 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 6.0 MiB/s wr, 534 op/s; 1.1 MiB/s, 17 objects/s recovering 2024-09-13T22:38:16.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:16 smithi118 ceph-mon[51566]: pgmap v147: 97 pgs: 97 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 6.0 MiB/s wr, 534 op/s; 1.1 MiB/s, 17 objects/s recovering 2024-09-13T22:38:17.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 193543 9.35 MB/sec execute 405 sec latency 47.638 ms 2024-09-13T22:38:18.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 193895 9.35 MB/sec execute 406 sec latency 45.192 ms 2024-09-13T22:38:18.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:18 smithi105 ceph-mon[65554]: pgmap v148: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.4 MiB/s wr, 515 op/s; 23 KiB/s, 5 objects/s recovering 2024-09-13T22:38:18.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:18 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T22:38:18.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:18 smithi118 ceph-mon[51566]: pgmap v148: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.4 MiB/s wr, 515 op/s; 23 KiB/s, 5 objects/s recovering 2024-09-13T22:38:18.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:18 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T22:38:19.409 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:19 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T22:38:19.409 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:19 smithi105 ceph-mon[65554]: Upgrade: osd.3 is safe to restart 2024-09-13T22:38:19.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 194164 9.35 MB/sec execute 407 sec latency 52.707 ms 2024-09-13T22:38:19.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:19 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T22:38:19.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:19 smithi118 ceph-mon[51566]: Upgrade: osd.3 is safe to restart 2024-09-13T22:38:20.417 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:20 smithi105 ceph-mon[65554]: pgmap v149: 97 pgs: 97 active+clean; 361 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 548 op/s; 20 KiB/s, 2 objects/s recovering 2024-09-13T22:38:20.417 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:20.417 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T22:38:20.417 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:38:20.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 194532 9.35 MB/sec execute 408 sec latency 47.734 ms 2024-09-13T22:38:20.748 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:20 smithi118 ceph-mon[51566]: pgmap v149: 97 pgs: 97 active+clean; 361 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 548 op/s; 20 KiB/s, 2 objects/s recovering 2024-09-13T22:38:20.748 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:20.748 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T22:38:20.748 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:38:21.433 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:21 smithi105 ceph-mon[65554]: Upgrade: Updating osd.3 2024-09-13T22:38:21.433 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:21 smithi105 ceph-mon[65554]: Deploying daemon osd.3 on smithi105 2024-09-13T22:38:21.433 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:21 smithi105 ceph-mon[65554]: pgmap v150: 97 pgs: 97 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.8 MiB/s wr, 688 op/s 2024-09-13T22:38:21.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 194760 9.34 MB/sec execute 409 sec latency 79.337 ms 2024-09-13T22:38:21.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:21 smithi118 ceph-mon[51566]: Upgrade: Updating osd.3 2024-09-13T22:38:21.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:21 smithi118 ceph-mon[51566]: Deploying daemon osd.3 on smithi105 2024-09-13T22:38:21.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:21 smithi118 ceph-mon[51566]: pgmap v150: 97 pgs: 97 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.8 MiB/s wr, 688 op/s 2024-09-13T22:38:22.446 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:38:22.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 195000 9.33 MB/sec execute 410 sec latency 60.243 ms 2024-09-13T22:38:23.257 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:38:23.257 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (6m) 26s ago 15m 17.6M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:38:23.257 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (2m) 26s ago 15m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:38:23.257 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (2m) 2m ago 14m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:38:23.257 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (6m) 26s ago 15m 45.6M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:38:23.257 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (9m) 26s ago 9m 38.5M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:38:23.257 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (9m) 2m ago 9m 14.3M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:38:23.257 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (5m) 26s ago 16m 504M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:38:23.257 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (4m) 2m ago 14m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (4m) 26s ago 16m 49.3M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (3m) 2m ago 14m 36.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (9m) 26s ago 9m 274M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (8m) 2m ago 8m 81.1M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (5m) 26s ago 15m 8502k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (5m) 2m ago 14m 14.5M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (2m) 26s ago 13m 267M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (83s) 26s ago 13m 216M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (37s) 26s ago 12m 12.1M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (12m) 26s ago 12m 593M 1936M 17.2.0 e1d6a67b021e dcdd39044b84 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (12m) 2m ago 12m 496M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (11m) 2m ago 11m 551M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:38:23.258 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (11m) 2m ago 11m 479M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:38:23.259 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (10m) 2m ago 10m 381M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:38:23.259 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (4m) 26s ago 14m 60.9M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:38:23.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 195339 9.33 MB/sec execute 411 sec latency 68.432 ms 2024-09-13T22:38:23.786 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:38:23.786 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:38:23.786 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:38:23.786 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:38:23.786 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:38:23.786 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:38:23.786 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:38:23.786 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 3 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 7 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:38:23.787 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:38:24.025 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:23 smithi105 ceph-mon[65554]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:24.025 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:23 smithi105 ceph-mon[65554]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:24.025 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:23 smithi105 ceph-mon[65554]: pgmap v151: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 531 op/s 2024-09-13T22:38:24.025 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:23 smithi105 ceph-mon[65554]: from='client.34318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:24.026 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:23 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/1479895821' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:24.200 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:38:24.200 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:38:24.200 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:38:24.200 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:38:24.200 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:38:24.201 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:38:24.201 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:38:24.201 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:38:24.201 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:38:24.201 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "9/23 daemons upgraded", 2024-09-13T22:38:24.201 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:38:24.201 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:38:24.201 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:38:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:23 smithi118 ceph-mon[51566]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:23 smithi118 ceph-mon[51566]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:23 smithi118 ceph-mon[51566]: pgmap v151: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 531 op/s 2024-09-13T22:38:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:23 smithi118 ceph-mon[51566]: from='client.34318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:23 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/1479895821' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:24.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 195610 9.33 MB/sec execute 412 sec latency 42.937 ms 2024-09-13T22:38:24.881 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:38:25.085 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:24 smithi105 ceph-mon[65554]: from='client.34326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:25.085 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:24 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/4149904287' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:38:25.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:24 smithi118 ceph-mon[51566]: from='client.34326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:25.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:24 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/4149904287' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:38:25.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 195856 9.33 MB/sec execute 413 sec latency 50.591 ms 2024-09-13T22:38:25.853 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:25 smithi105 systemd[1]: Stopping Ceph osd.3 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:38:26.110 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:25 smithi105 ceph-mon[65554]: pgmap v152: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 582 op/s 2024-09-13T22:38:26.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:25 smithi118 ceph-mon[51566]: pgmap v152: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 582 op/s 2024-09-13T22:38:26.363 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:26 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[44676]: 2024-09-13T22:38:26.156+0000 7f094acd8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:38:26.363 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:26 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[44676]: 2024-09-13T22:38:26.156+0000 7f094acd8700 -1 osd.3 105 *** Got signal Terminated *** 2024-09-13T22:38:26.363 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:26 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[44676]: 2024-09-13T22:38:26.156+0000 7f094acd8700 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:38:26.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 196250 9.34 MB/sec execute 414 sec latency 52.805 ms 2024-09-13T22:38:27.116 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:26 smithi105 ceph-mon[65554]: osd.3 marked itself down and dead 2024-09-13T22:38:27.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:26 smithi118 ceph-mon[51566]: osd.3 marked itself down and dead 2024-09-13T22:38:27.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 196634 9.33 MB/sec execute 415 sec latency 631.831 ms 2024-09-13T22:38:27.895 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:27 smithi105 podman[79194]: 2024-09-13 22:38:27.632399112 +0000 UTC m=+1.618672928 container died dcdd39044b847e2798518aee4bae54c38f0ecd4e0f1a2223924701698393b747 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, ceph=True, vendor=Red Hat, Inc., architecture=x86_64, build-date=2022-05-03T08:36:31.336870) 2024-09-13T22:38:28.200 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:27 smithi105 ceph-mon[65554]: pgmap v153: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 521 op/s 2024-09-13T22:38:28.200 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:27 smithi105 ceph-mon[65554]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:38:28.200 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:27 smithi105 ceph-mon[65554]: osdmap e106: 8 total, 7 up, 8 in 2024-09-13T22:38:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:27 smithi118 ceph-mon[51566]: pgmap v153: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 521 op/s 2024-09-13T22:38:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:27 smithi118 ceph-mon[51566]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:38:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:27 smithi118 ceph-mon[51566]: osdmap e106: 8 total, 7 up, 8 in 2024-09-13T22:38:28.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 196897 9.32 MB/sec execute 416 sec latency 1036.586 ms 2024-09-13T22:38:28.702 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:28 smithi105 podman[79194]: 2024-09-13 22:38:28.506300937 +0000 UTC m=+2.492574753 container cleanup dcdd39044b847e2798518aee4bae54c38f0ecd4e0f1a2223924701698393b747 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, version=8, io.k8s.display-name=CentOS Stream 8, release=754, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, RELEASE=HEAD) 2024-09-13T22:38:28.702 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:28 smithi105 bash[79194]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3 2024-09-13T22:38:28.954 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:28 smithi105 ceph-mon[65554]: osdmap e107: 8 total, 7 up, 8 in 2024-09-13T22:38:28.954 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:28.954 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:38:28.954 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:28 smithi105 podman[79212]: 2024-09-13 22:38:28.951722235 +0000 UTC m=+1.322647261 container remove dcdd39044b847e2798518aee4bae54c38f0ecd4e0f1a2223924701698393b747 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., architecture=x86_64, io.buildah.version=1.19.8, vcs-type=git, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, RELEASE=HEAD, name=centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T22:38:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:28 smithi118 ceph-mon[51566]: osdmap e107: 8 total, 7 up, 8 in 2024-09-13T22:38:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:38:29.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 197422 9.34 MB/sec execute 417 sec latency 71.698 ms 2024-09-13T22:38:29.707 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:29 smithi105 podman[79295]: 2024-09-13 22:38:29.472516645 +0000 UTC m=+0.023774956 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:38:29.958 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:29 smithi105 podman[79295]: 2024-09-13 22:38:29.757658552 +0000 UTC m=+0.308916842 container create 31e8ffe8efe90fe6f03d7e048db1d0b3bb4e8700ca6c5e16c369c6e2b4cf68b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:38:29.958 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:29 smithi105 ceph-mon[65554]: pgmap v156: 97 pgs: 14 peering, 11 stale+active+clean, 72 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.4 MiB/s wr, 286 op/s 2024-09-13T22:38:29.958 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:29 smithi105 ceph-mon[65554]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T22:38:30.214 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:30 smithi105 podman[79295]: 2024-09-13 22:38:30.203386236 +0000 UTC m=+0.754644532 container init 31e8ffe8efe90fe6f03d7e048db1d0b3bb4e8700ca6c5e16c369c6e2b4cf68b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T22:38:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:29 smithi118 ceph-mon[51566]: pgmap v156: 97 pgs: 14 peering, 11 stale+active+clean, 72 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.4 MiB/s wr, 286 op/s 2024-09-13T22:38:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:29 smithi118 ceph-mon[51566]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T22:38:30.466 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:30 smithi105 podman[79295]: 2024-09-13 22:38:30.215019984 +0000 UTC m=+0.766278275 container start 31e8ffe8efe90fe6f03d7e048db1d0b3bb4e8700ca6c5e16c369c6e2b4cf68b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:38:30.466 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:30 smithi105 podman[79295]: 2024-09-13 22:38:30.437082641 +0000 UTC m=+0.988340944 container attach 31e8ffe8efe90fe6f03d7e048db1d0b3bb4e8700ca6c5e16c369c6e2b4cf68b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T22:38:30.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 198166 9.35 MB/sec execute 418 sec latency 54.136 ms 2024-09-13T22:38:30.720 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:30 smithi105 podman[79295]: 2024-09-13 22:38:30.567081466 +0000 UTC m=+1.118339757 container died 31e8ffe8efe90fe6f03d7e048db1d0b3bb4e8700ca6c5e16c369c6e2b4cf68b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:38:31.232 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:30 smithi105 ceph-mon[65554]: Health check failed: Degraded data redundancy: 100/1668 objects degraded (5.995%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:31 smithi118 ceph-mon[51566]: Health check failed: Degraded data redundancy: 100/1668 objects degraded (5.995%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:31.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 198824 9.39 MB/sec execute 419 sec latency 33.924 ms 2024-09-13T22:38:32.253 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:32 smithi105 podman[79318]: 2024-09-13 22:38:32.010680212 +0000 UTC m=+1.436738981 container remove 31e8ffe8efe90fe6f03d7e048db1d0b3bb4e8700ca6c5e16c369c6e2b4cf68b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T22:38:32.253 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:32 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T22:38:32.253 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:32 smithi105 systemd[1]: Stopped Ceph osd.3 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:38:32.253 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:32 smithi105 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.3.service: Consumed 3min 46.150s CPU time. 2024-09-13T22:38:32.253 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:32 smithi105 ceph-mon[65554]: pgmap v157: 97 pgs: 7 active+undersized, 14 peering, 3 stale+active+clean, 11 active+undersized+degraded, 62 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s wr, 575 op/s; 100/1668 objects degraded (5.995%) 2024-09-13T22:38:32.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:32 smithi118 ceph-mon[51566]: pgmap v157: 97 pgs: 7 active+undersized, 14 peering, 3 stale+active+clean, 11 active+undersized+degraded, 62 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s wr, 575 op/s; 100/1668 objects degraded (5.995%) 2024-09-13T22:38:32.503 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:32 smithi105 systemd[1]: Starting Ceph osd.3 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:38:32.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 199361 9.40 MB/sec execute 420 sec latency 31.930 ms 2024-09-13T22:38:32.759 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:32 smithi105 podman[79410]: 2024-09-13 22:38:32.573807043 +0000 UTC m=+0.020047289 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:38:32.759 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:32 smithi105 podman[79410]: 2024-09-13 22:38:32.714561669 +0000 UTC m=+0.160801907 container create d29823b5a985112cee49bae00f244b0662d23e7f3dd28364fc45837080a91404 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T22:38:33.264 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:33 smithi105 ceph-mon[65554]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-13T22:38:33.264 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 podman[79410]: 2024-09-13 22:38:33.051144378 +0000 UTC m=+0.497384629 container init d29823b5a985112cee49bae00f244b0662d23e7f3dd28364fc45837080a91404 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:38:33.264 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 podman[79410]: 2024-09-13 22:38:33.057263444 +0000 UTC m=+0.503503683 container start d29823b5a985112cee49bae00f244b0662d23e7f3dd28364fc45837080a91404 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-ecab7be) 2024-09-13T22:38:33.264 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate[79424]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:38:33.264 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 bash[79410]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:38:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:33 smithi118 ceph-mon[51566]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-13T22:38:33.516 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 podman[79410]: 2024-09-13 22:38:33.263801853 +0000 UTC m=+0.710042094 container attach d29823b5a985112cee49bae00f244b0662d23e7f3dd28364fc45837080a91404 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-13T22:38:33.516 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate[79424]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:38:33.516 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 bash[79410]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:38:33.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 199692 9.40 MB/sec execute 421 sec latency 38.634 ms 2024-09-13T22:38:34.043 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate[79424]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T22:38:34.043 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 bash[79410]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T22:38:34.043 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate[79424]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T22:38:34.043 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:33 smithi105 bash[79410]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T22:38:34.294 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate[79424]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 bash[79410]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate[79424]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 bash[79410]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate[79424]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 bash[79410]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate[79424]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 bash[79410]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate[79424]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 bash[79410]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T22:38:34.295 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:34 smithi105 podman[79410]: 2024-09-13 22:38:34.235617726 +0000 UTC m=+1.681857971 container died d29823b5a985112cee49bae00f244b0662d23e7f3dd28364fc45837080a91404 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T22:38:34.296 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:34 smithi105 ceph-mon[65554]: pgmap v158: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 428 op/s; 232/1719 objects degraded (13.496%) 2024-09-13T22:38:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:34 smithi118 ceph-mon[51566]: pgmap v158: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 428 op/s; 232/1719 objects degraded (13.496%) 2024-09-13T22:38:34.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 199990 9.40 MB/sec execute 422 sec latency 69.561 ms 2024-09-13T22:38:35.325 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:35 smithi105 podman[79564]: 2024-09-13 22:38:35.301299611 +0000 UTC m=+1.054889736 container remove d29823b5a985112cee49bae00f244b0662d23e7f3dd28364fc45837080a91404 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:38:35.576 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:35 smithi105 podman[79622]: 2024-09-13 22:38:35.447785287 +0000 UTC m=+0.020538756 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:38:35.576 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:35 smithi105 podman[79622]: 2024-09-13 22:38:35.560699976 +0000 UTC m=+0.133453430 container create 3359b8a2d07315f2c811916b1d1d9730ab7b20e7ba2a4d73597373588d0adf23 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:38:35.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 200324 9.39 MB/sec execute 423 sec latency 49.164 ms 2024-09-13T22:38:36.082 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:35 smithi105 podman[79622]: 2024-09-13 22:38:35.932438892 +0000 UTC m=+0.505192343 container init 3359b8a2d07315f2c811916b1d1d9730ab7b20e7ba2a4d73597373588d0adf23 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, io.buildah.version=1.37.2) 2024-09-13T22:38:36.082 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:35 smithi105 podman[79622]: 2024-09-13 22:38:35.946100101 +0000 UTC m=+0.518853554 container start 3359b8a2d07315f2c811916b1d1d9730ab7b20e7ba2a4d73597373588d0adf23 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:38:36.082 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:35 smithi105 ceph-osd[79639]: -- 172.21.15.105:0/3760741203 <== mon.0 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557b873821e0 con 0x557b8656fc00 2024-09-13T22:38:36.082 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:36 smithi105 ceph-mon[65554]: pgmap v159: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 7.5 MiB/s wr, 770 op/s; 237/1806 objects degraded (13.123%) 2024-09-13T22:38:36.336 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:36 smithi105 bash[79622]: 3359b8a2d07315f2c811916b1d1d9730ab7b20e7ba2a4d73597373588d0adf23 2024-09-13T22:38:36.336 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:36 smithi105 systemd[1]: Started Ceph osd.3 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:38:36.346 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:36 smithi118 ceph-mon[51566]: pgmap v159: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 7.5 MiB/s wr, 770 op/s; 237/1806 objects degraded (13.123%) 2024-09-13T22:38:36.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 200589 9.39 MB/sec execute 424 sec latency 64.116 ms 2024-09-13T22:38:37.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:37 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:37.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:37 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:37.340 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:37 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:38:37.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 201048 9.39 MB/sec execute 425 sec latency 42.519 ms 2024-09-13T22:38:37.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:37 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:37.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:37 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:37.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:37 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:38:38.115 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:38 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[79635]: 2024-09-13T22:38:38.057+0000 7fd1b017f640 -1 Falling back to public interface 2024-09-13T22:38:38.171 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:38 smithi105 ceph-mon[65554]: pgmap v160: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 6.5 MiB/s wr, 643 op/s; 219/1749 objects degraded (12.521%) 2024-09-13T22:38:38.171 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:38 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 219/1749 objects degraded (12.521%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:38.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 201434 9.38 MB/sec execute 426 sec latency 55.682 ms 2024-09-13T22:38:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:38 smithi118 ceph-mon[51566]: pgmap v160: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 6.5 MiB/s wr, 643 op/s; 219/1749 objects degraded (12.521%) 2024-09-13T22:38:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:38 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 219/1749 objects degraded (12.521%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:39.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 201775 9.38 MB/sec execute 427 sec latency 45.016 ms 2024-09-13T22:38:40.192 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:40 smithi105 ceph-mon[65554]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.0 MiB/s wr, 594 op/s; 214/1731 objects degraded (12.363%) 2024-09-13T22:38:40.579 INFO:teuthology.orchestra.run.smithi105.stdout: 5 202126 9.39 MB/sec execute 428 sec latency 44.184 ms 2024-09-13T22:38:40.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:40 smithi118 ceph-mon[51566]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.0 MiB/s wr, 594 op/s; 214/1731 objects degraded (12.363%) 2024-09-13T22:38:41.580 INFO:teuthology.orchestra.run.smithi105.stdout: 5 202623 9.41 MB/sec execute 429 sec latency 51.178 ms 2024-09-13T22:38:42.204 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:42 smithi105 ceph-mon[65554]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.0 MiB/s wr, 727 op/s; 211/1725 objects degraded (12.232%) 2024-09-13T22:38:42.580 INFO:teuthology.orchestra.run.smithi105.stdout: 5 202919 9.41 MB/sec execute 430 sec latency 70.186 ms 2024-09-13T22:38:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:42 smithi118 ceph-mon[51566]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.0 MiB/s wr, 727 op/s; 211/1725 objects degraded (12.232%) 2024-09-13T22:38:43.241 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:43 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 217/1716 objects degraded (12.646%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:43.580 INFO:teuthology.orchestra.run.smithi105.stdout: 5 203144 9.40 MB/sec execute 431 sec latency 46.758 ms 2024-09-13T22:38:43.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:43 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 217/1716 objects degraded (12.646%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:44.260 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:44 smithi105 ceph-mon[65554]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 547 op/s; 217/1716 objects degraded (12.646%) 2024-09-13T22:38:44.260 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:44.260 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:38:44.580 INFO:teuthology.orchestra.run.smithi105.stdout: 5 203355 9.39 MB/sec execute 432 sec latency 93.868 ms 2024-09-13T22:38:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:44 smithi118 ceph-mon[51566]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 547 op/s; 217/1716 objects degraded (12.646%) 2024-09-13T22:38:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:38:45.389 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:45 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[79635]: 2024-09-13T22:38:45.297+0000 7fd1b017f640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-13T22:38:45.580 INFO:teuthology.orchestra.run.smithi105.stdout: 5 203565 9.38 MB/sec execute 433 sec latency 83.153 ms 2024-09-13T22:38:46.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:46 smithi105 ceph-mon[65554]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.8 MiB/s wr, 672 op/s; 219/1701 objects degraded (12.875%) 2024-09-13T22:38:46.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:46.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:46.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:46.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:46 smithi118 ceph-mon[51566]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.8 MiB/s wr, 672 op/s; 219/1701 objects degraded (12.875%) 2024-09-13T22:38:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:46.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:46.572 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:46 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[79635]: 2024-09-13T22:38:46.352+0000 7fd1b017f640 -1 osd.3 105 log_to_monitors true 2024-09-13T22:38:46.580 INFO:teuthology.orchestra.run.smithi105.stdout: 5 203853 9.37 MB/sec execute 434 sec latency 64.948 ms 2024-09-13T22:38:47.087 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:47 smithi105 ceph-mon[65554]: from='osd.3 [v2:172.21.15.105:6826/109246858,v1:172.21.15.105:6827/109246858]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T22:38:47.338 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:38:47 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[79635]: 2024-09-13T22:38:47.090+0000 7fd1a7f2a640 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:38:47.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:47 smithi118 ceph-mon[51566]: from='osd.3 [v2:172.21.15.105:6826/109246858,v1:172.21.15.105:6827/109246858]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T22:38:47.580 INFO:teuthology.orchestra.run.smithi105.stdout: 5 204112 9.36 MB/sec execute 435 sec latency 47.070 ms 2024-09-13T22:38:48.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:48 smithi105 ceph-mon[65554]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 423 op/s; 212/1680 objects degraded (12.619%) 2024-09-13T22:38:48.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:48 smithi105 ceph-mon[65554]: from='osd.3 [v2:172.21.15.105:6826/109246858,v1:172.21.15.105:6827/109246858]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T22:38:48.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:48 smithi105 ceph-mon[65554]: osdmap e108: 8 total, 7 up, 8 in 2024-09-13T22:38:48.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:48 smithi105 ceph-mon[65554]: from='osd.3 [v2:172.21.15.105:6826/109246858,v1:172.21.15.105:6827/109246858]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:38:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:48 smithi118 ceph-mon[51566]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 423 op/s; 212/1680 objects degraded (12.619%) 2024-09-13T22:38:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:48 smithi118 ceph-mon[51566]: from='osd.3 [v2:172.21.15.105:6826/109246858,v1:172.21.15.105:6827/109246858]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T22:38:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:48 smithi118 ceph-mon[51566]: osdmap e108: 8 total, 7 up, 8 in 2024-09-13T22:38:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:48 smithi118 ceph-mon[51566]: from='osd.3 [v2:172.21.15.105:6826/109246858,v1:172.21.15.105:6827/109246858]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-13T22:38:48.580 INFO:teuthology.orchestra.run.smithi105.stdout: 5 204248 9.35 MB/sec execute 436 sec latency 551.800 ms 2024-09-13T22:38:49.317 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:49 smithi105 ceph-mon[65554]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:38:49.318 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:49 smithi105 ceph-mon[65554]: osd.3 [v2:172.21.15.105:6826/109246858,v1:172.21.15.105:6827/109246858] boot 2024-09-13T22:38:49.318 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:49 smithi105 ceph-mon[65554]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T22:38:49.318 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:38:49.318 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:49 smithi105 ceph-mon[65554]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T22:38:49.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:49 smithi118 ceph-mon[51566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:38:49.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:49 smithi118 ceph-mon[51566]: osd.3 [v2:172.21.15.105:6826/109246858,v1:172.21.15.105:6827/109246858] boot 2024-09-13T22:38:49.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:49 smithi118 ceph-mon[51566]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T22:38:49.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T22:38:49.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:49 smithi118 ceph-mon[51566]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T22:38:49.580 INFO:teuthology.orchestra.run.smithi105.stdout: 5 204248 9.33 MB/sec execute 437 sec latency 1551.828 ms 2024-09-13T22:38:50.289 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:50 smithi105 ceph-mon[65554]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 3.9 MiB/s wr, 355 op/s; 211/1674 objects degraded (12.605%) 2024-09-13T22:38:50.289 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:50 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 212/1680 objects degraded (12.619%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:50.289 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:50 smithi105 ceph-mon[65554]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T22:38:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:50 smithi118 ceph-mon[51566]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 3.9 MiB/s wr, 355 op/s; 211/1674 objects degraded (12.605%) 2024-09-13T22:38:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:50 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 212/1680 objects degraded (12.619%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:50 smithi118 ceph-mon[51566]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T22:38:50.584 INFO:teuthology.orchestra.run.smithi105.stdout: 5 204362 9.32 MB/sec execute 438 sec latency 2152.299 ms 2024-09-13T22:38:51.586 INFO:teuthology.orchestra.run.smithi105.stdout: 5 204567 9.31 MB/sec execute 439 sec latency 552.187 ms 2024-09-13T22:38:52.291 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: pgmap v171: 97 pgs: 6 remapped+peering, 9 active+undersized, 20 active+undersized+degraded, 62 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 2.3 MiB/s wr, 262 op/s; 138/1620 objects degraded (8.519%); 22 B/s, 0 objects/s recovering 2024-09-13T22:38:52.291 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:52.292 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:52 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T22:38:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: pgmap v171: 97 pgs: 6 remapped+peering, 9 active+undersized, 20 active+undersized+degraded, 62 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 2.3 MiB/s wr, 262 op/s; 138/1620 objects degraded (8.519%); 22 B/s, 0 objects/s recovering 2024-09-13T22:38:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T22:38:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:38:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:38:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:52.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:52.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:38:52.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:52.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:52.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:52.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:52 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T22:38:52.587 INFO:teuthology.orchestra.run.smithi105.stdout: 5 204720 9.29 MB/sec execute 440 sec latency 1034.573 ms 2024-09-13T22:38:53.109 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:53 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T22:38:53.109 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:53 smithi105 ceph-mon[65554]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T22:38:53.109 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:53 smithi105 ceph-mon[65554]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:38:53.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:53 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T22:38:53.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:53 smithi118 ceph-mon[51566]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-13T22:38:53.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:53 smithi118 ceph-mon[51566]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T22:38:53.594 INFO:teuthology.orchestra.run.smithi105.stdout: 5 204983 9.28 MB/sec execute 441 sec latency 547.251 ms 2024-09-13T22:38:54.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:54 smithi105 ceph-mon[65554]: pgmap v174: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 5 remapped+peering, 83 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.9 MiB/s wr, 353 op/s; 140/1584 objects degraded (8.838%); 667 KiB/s, 23 objects/s recovering 2024-09-13T22:38:54.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:54 smithi105 ceph-mon[65554]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T22:38:54.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:54 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 140/1584 objects degraded (8.838%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:54 smithi118 ceph-mon[51566]: pgmap v174: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 5 remapped+peering, 83 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.9 MiB/s wr, 353 op/s; 140/1584 objects degraded (8.838%); 667 KiB/s, 23 objects/s recovering 2024-09-13T22:38:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:54 smithi118 ceph-mon[51566]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T22:38:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:54 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 140/1584 objects degraded (8.838%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:54.594 INFO:teuthology.orchestra.run.smithi105.stdout: 5 205229 9.28 MB/sec execute 442 sec latency 1026.012 ms 2024-09-13T22:38:55.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:55 smithi105 ceph-mon[65554]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T22:38:55.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:55 smithi118 ceph-mon[51566]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T22:38:55.427 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:38:55.594 INFO:teuthology.orchestra.run.smithi105.stdout: 5 205644 9.29 MB/sec execute 443 sec latency 40.328 ms 2024-09-13T22:38:56.078 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:56 smithi105 ceph-mon[65554]: pgmap v177: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped+peering, 87 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.2 MiB/s wr, 259 op/s; 140/1551 objects degraded (9.026%); 1.6 MiB/s, 48 objects/s recovering 2024-09-13T22:38:56.078 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:56 smithi105 ceph-mon[65554]: from='client.44241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:56.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:56 smithi118 ceph-mon[51566]: pgmap v177: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped+peering, 87 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.2 MiB/s wr, 259 op/s; 140/1551 objects degraded (9.026%); 1.6 MiB/s, 48 objects/s recovering 2024-09-13T22:38:56.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:56 smithi118 ceph-mon[51566]: from='client.44241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:56.413 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:38:56.413 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (6m) 11s ago 15m 17.7M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:38:56.413 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (3m) 11s ago 15m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:38:56.413 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (3m) 3m ago 15m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:38:56.413 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (6m) 11s ago 15m 46.4M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:38:56.413 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (9m) 11s ago 9m 38.6M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:38:56.413 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (9m) 3m ago 9m 14.3M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:38:56.413 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (5m) 11s ago 17m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (5m) 3m ago 15m 439M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (4m) 11s ago 17m 50.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (4m) 3m ago 15m 36.4M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (9m) 11s ago 9m 279M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (9m) 3m ago 9m 81.1M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (6m) 11s ago 15m 8002k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (6m) 3m ago 15m 14.5M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (2m) 11s ago 14m 327M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (116s) 11s ago 13m 270M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:38:56.414 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (70s) 11s ago 13m 189M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:38:56.415 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (20s) 11s ago 13m 12.1M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:38:56.415 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (12m) 3m ago 12m 496M 2704M 17.2.0 e1d6a67b021e 4bdb74663abe 2024-09-13T22:38:56.415 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (12m) 3m ago 12m 551M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:38:56.415 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (11m) 3m ago 11m 479M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:38:56.415 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (11m) 3m ago 11m 381M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:38:56.415 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (5m) 11s ago 14m 61.0M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:38:56.594 INFO:teuthology.orchestra.run.smithi105.stdout: 5 206150 9.30 MB/sec execute 444 sec latency 26.957 ms 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T22:38:57.033 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:38:57.034 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:38:57.034 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:38:57.034 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:38:57.034 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:38:57.034 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T22:38:57.034 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T22:38:57.034 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:38:57.034 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:38:57.089 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:57 smithi105 ceph-mon[65554]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:57.089 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:57 smithi105 ceph-mon[65554]: from='client.44249 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:57.089 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:57 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/3072839276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:57.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:57 smithi118 ceph-mon[51566]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:57.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:57 smithi118 ceph-mon[51566]: from='client.44249 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:57.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:57 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/3072839276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:38:57.565 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:38:57.565 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:38:57.565 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:38:57.565 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:38:57.565 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:38:57.565 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:38:57.566 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:38:57.566 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:38:57.566 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:38:57.566 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "10/23 daemons upgraded", 2024-09-13T22:38:57.566 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:38:57.566 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:38:57.566 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:38:57.594 INFO:teuthology.orchestra.run.smithi105.stdout: 5 206474 9.31 MB/sec execute 445 sec latency 57.367 ms 2024-09-13T22:38:58.110 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:58 smithi105 ceph-mon[65554]: pgmap v178: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped+peering, 90 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.4 MiB/s wr, 211 op/s; 60/1590 objects degraded (3.774%); 1.4 MiB/s, 46 objects/s recovering 2024-09-13T22:38:58.210 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_WARN Degraded data redundancy: 60/1590 objects degraded (3.774%), 2 pgs degraded 2024-09-13T22:38:58.210 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 60/1590 objects degraded (3.774%), 2 pgs degraded 2024-09-13T22:38:58.210 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.8 is active+recovery_wait+undersized+degraded+remapped, acting [1,7] 2024-09-13T22:38:58.210 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.b is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2024-09-13T22:38:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:58 smithi118 ceph-mon[51566]: pgmap v178: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 remapped+peering, 90 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.4 MiB/s wr, 211 op/s; 60/1590 objects degraded (3.774%); 1.4 MiB/s, 46 objects/s recovering 2024-09-13T22:38:58.594 INFO:teuthology.orchestra.run.smithi105.stdout: 5 206774 9.31 MB/sec execute 446 sec latency 47.220 ms 2024-09-13T22:38:59.128 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:59 smithi105 ceph-mon[65554]: from='client.44257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:59.128 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:59 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/4097299460' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:38:59.128 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:59.129 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:38:59.129 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:38:59 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 60/1590 objects degraded (3.774%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:59 smithi118 ceph-mon[51566]: from='client.44257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:38:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:59 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/4097299460' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:38:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:38:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:38:59.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:38:59 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 60/1590 objects degraded (3.774%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T22:38:59.594 INFO:teuthology.orchestra.run.smithi105.stdout: 5 207158 9.31 MB/sec execute 447 sec latency 41.988 ms 2024-09-13T22:39:00.137 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:00 smithi105 ceph-mon[65554]: pgmap v179: 97 pgs: 97 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 3.4 MiB/s wr, 408 op/s; 1.8 MiB/s, 43 objects/s recovering 2024-09-13T22:39:00.137 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:00 smithi105 ceph-mon[65554]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/1590 objects degraded (3.774%), 2 pgs degraded) 2024-09-13T22:39:00.137 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:00 smithi105 ceph-mon[65554]: Cluster is now healthy 2024-09-13T22:39:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:00 smithi118 ceph-mon[51566]: pgmap v179: 97 pgs: 97 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 3.4 MiB/s wr, 408 op/s; 1.8 MiB/s, 43 objects/s recovering 2024-09-13T22:39:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:00 smithi118 ceph-mon[51566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 60/1590 objects degraded (3.774%), 2 pgs degraded) 2024-09-13T22:39:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:00 smithi118 ceph-mon[51566]: Cluster is now healthy 2024-09-13T22:39:00.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 207574 9.30 MB/sec execute 448 sec latency 30.297 ms 2024-09-13T22:39:01.594 INFO:teuthology.orchestra.run.smithi105.stdout: 5 207934 9.30 MB/sec execute 449 sec latency 55.479 ms 2024-09-13T22:39:02.179 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:02 smithi105 ceph-mon[65554]: pgmap v180: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 4.6 MiB/s wr, 460 op/s; 1.3 MiB/s, 33 objects/s recovering 2024-09-13T22:39:02.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:02 smithi118 ceph-mon[51566]: pgmap v180: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 4.6 MiB/s wr, 460 op/s; 1.3 MiB/s, 33 objects/s recovering 2024-09-13T22:39:02.594 INFO:teuthology.orchestra.run.smithi105.stdout: 5 208458 9.30 MB/sec execute 450 sec latency 52.775 ms 2024-09-13T22:39:03.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 208967 9.32 MB/sec execute 451 sec latency 33.830 ms 2024-09-13T22:39:04.192 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:04 smithi105 ceph-mon[65554]: pgmap v181: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 423 op/s; 1.0 MiB/s, 27 objects/s recovering 2024-09-13T22:39:04.192 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:04 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:04.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:04 smithi118 ceph-mon[51566]: pgmap v181: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 423 op/s; 1.0 MiB/s, 27 objects/s recovering 2024-09-13T22:39:04.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:04 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:04.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 209512 9.34 MB/sec execute 452 sec latency 57.535 ms 2024-09-13T22:39:05.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 209980 9.35 MB/sec execute 453 sec latency 42.349 ms 2024-09-13T22:39:06.211 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:06 smithi105 ceph-mon[65554]: pgmap v182: 97 pgs: 97 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.1 MiB/s wr, 637 op/s; 951 KiB/s, 24 objects/s recovering 2024-09-13T22:39:06.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:06 smithi118 ceph-mon[51566]: pgmap v182: 97 pgs: 97 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.1 MiB/s wr, 637 op/s; 951 KiB/s, 24 objects/s recovering 2024-09-13T22:39:06.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 210382 9.36 MB/sec execute 454 sec latency 41.199 ms 2024-09-13T22:39:07.216 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T22:39:07.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T22:39:07.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 210818 9.36 MB/sec execute 455 sec latency 38.257 ms 2024-09-13T22:39:08.234 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:08 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T22:39:08.234 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:08 smithi105 ceph-mon[65554]: Upgrade: osd.4 is safe to restart 2024-09-13T22:39:08.234 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:08 smithi105 ceph-mon[65554]: pgmap v183: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.7 MiB/s wr, 573 op/s; 516 KiB/s, 14 objects/s recovering 2024-09-13T22:39:08.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:08 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T22:39:08.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:08 smithi118 ceph-mon[51566]: Upgrade: osd.4 is safe to restart 2024-09-13T22:39:08.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:08 smithi118 ceph-mon[51566]: pgmap v183: 97 pgs: 97 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.7 MiB/s wr, 573 op/s; 516 KiB/s, 14 objects/s recovering 2024-09-13T22:39:08.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 211206 9.35 MB/sec execute 456 sec latency 33.955 ms 2024-09-13T22:39:09.501 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:09 smithi105 ceph-mon[65554]: Upgrade: Updating osd.4 2024-09-13T22:39:09.501 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:09.501 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T22:39:09.501 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:39:09.501 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:09 smithi105 ceph-mon[65554]: Deploying daemon osd.4 on smithi118 2024-09-13T22:39:09.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 211666 9.36 MB/sec execute 457 sec latency 42.140 ms 2024-09-13T22:39:09.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:09 smithi118 ceph-mon[51566]: Upgrade: Updating osd.4 2024-09-13T22:39:09.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:09.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T22:39:09.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:39:09.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:09 smithi118 ceph-mon[51566]: Deploying daemon osd.4 on smithi118 2024-09-13T22:39:10.511 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:10 smithi105 ceph-mon[65554]: pgmap v184: 97 pgs: 97 active+clean; 361 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 670 op/s; 408 KiB/s, 7 objects/s recovering 2024-09-13T22:39:10.528 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:10 smithi118 ceph-mon[51566]: pgmap v184: 97 pgs: 97 active+clean; 361 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 670 op/s; 408 KiB/s, 7 objects/s recovering 2024-09-13T22:39:10.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 212186 9.36 MB/sec execute 458 sec latency 69.519 ms 2024-09-13T22:39:11.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 212553 9.36 MB/sec execute 459 sec latency 36.157 ms 2024-09-13T22:39:12.097 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:12 smithi118 systemd[1]: Stopping Ceph osd.4 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:39:12.408 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:12 smithi105 ceph-mon[65554]: pgmap v185: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.5 MiB/s wr, 593 op/s 2024-09-13T22:39:12.584 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:12 smithi118 ceph-mon[51566]: pgmap v185: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.5 MiB/s wr, 593 op/s 2024-09-13T22:39:12.584 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[29898]: 2024-09-13T22:39:12.369+0000 7fd0c942e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:39:12.584 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[29898]: 2024-09-13T22:39:12.369+0000 7fd0c942e700 -1 osd.4 115 *** Got signal Terminated *** 2024-09-13T22:39:12.584 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[29898]: 2024-09-13T22:39:12.369+0000 7fd0c942e700 -1 osd.4 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:39:12.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 212906 9.37 MB/sec execute 460 sec latency 56.644 ms 2024-09-13T22:39:13.418 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:13 smithi105 ceph-mon[65554]: osd.4 marked itself down and dead 2024-09-13T22:39:13.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 213499 9.39 MB/sec execute 461 sec latency 250.805 ms 2024-09-13T22:39:13.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:13 smithi118 ceph-mon[51566]: osd.4 marked itself down and dead 2024-09-13T22:39:14.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:13 smithi118 podman[55968]: 2024-09-13 22:39:13.916422142 +0000 UTC m=+1.677823172 container died 4bdb74663abeeb13490699f8f2c4b3d122110a87a839ba7b1418b7d307e0aa9a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.openshift.tags=base centos centos-stream, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, RELEASE=HEAD, release=754, io.k8s.display-name=CentOS Stream 8) 2024-09-13T22:39:14.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 213626 9.39 MB/sec execute 462 sec latency 1024.653 ms 2024-09-13T22:39:14.605 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:14 smithi105 ceph-mon[65554]: pgmap v186: 97 pgs: 97 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-09-13T22:39:14.605 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:14 smithi105 ceph-mon[65554]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:39:14.605 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:14 smithi105 ceph-mon[65554]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T22:39:14.605 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:14.605 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:39:14.807 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:14 smithi118 ceph-mon[51566]: pgmap v186: 97 pgs: 97 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-09-13T22:39:14.807 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:14 smithi118 ceph-mon[51566]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:39:14.807 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:14 smithi118 ceph-mon[51566]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T22:39:14.807 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:14.807 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:39:14.807 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:14 smithi118 podman[55968]: 2024-09-13 22:39:14.520749864 +0000 UTC m=+2.282150933 container cleanup 4bdb74663abeeb13490699f8f2c4b3d122110a87a839ba7b1418b7d307e0aa9a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4, vcs-type=git, GIT_CLEAN=True, RELEASE=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux ) 2024-09-13T22:39:14.807 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:14 smithi118 bash[55968]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4 2024-09-13T22:39:15.097 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:14 smithi118 podman[55994]: 2024-09-13 22:39:14.806342424 +0000 UTC m=+0.886156351 container remove 4bdb74663abeeb13490699f8f2c4b3d122110a87a839ba7b1418b7d307e0aa9a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, release=754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, version=8, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64) 2024-09-13T22:39:15.453 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:15 smithi118 ceph-mon[51566]: osdmap e117: 8 total, 7 up, 8 in 2024-09-13T22:39:15.453 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:15 smithi118 podman[56078]: 2024-09-13 22:39:15.049377454 +0000 UTC m=+0.018253258 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:39:15.454 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:15 smithi118 podman[56078]: 2024-09-13 22:39:15.184555822 +0000 UTC m=+0.153431613 container create e26464d283938bdee19b00cbfff8ebb2c5f9226f0d78b0802ea46a4563bed91f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be) 2024-09-13T22:39:15.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 213943 9.38 MB/sec execute 463 sec latency 44.275 ms 2024-09-13T22:39:15.614 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:15 smithi105 ceph-mon[65554]: osdmap e117: 8 total, 7 up, 8 in 2024-09-13T22:39:15.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:15 smithi118 podman[56078]: 2024-09-13 22:39:15.453181599 +0000 UTC m=+0.422057405 container init e26464d283938bdee19b00cbfff8ebb2c5f9226f0d78b0802ea46a4563bed91f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T22:39:15.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:15 smithi118 podman[56078]: 2024-09-13 22:39:15.457989821 +0000 UTC m=+0.426865615 container start e26464d283938bdee19b00cbfff8ebb2c5f9226f0d78b0802ea46a4563bed91f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T22:39:15.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:15 smithi118 podman[56078]: 2024-09-13 22:39:15.561752378 +0000 UTC m=+0.530628169 container attach e26464d283938bdee19b00cbfff8ebb2c5f9226f0d78b0802ea46a4563bed91f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T22:39:15.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:15 smithi118 podman[56078]: 2024-09-13 22:39:15.671489154 +0000 UTC m=+0.640364958 container died e26464d283938bdee19b00cbfff8ebb2c5f9226f0d78b0802ea46a4563bed91f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:39:16.386 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:16 smithi105 ceph-mon[65554]: pgmap v189: 97 pgs: 4 active+undersized, 9 stale+active+clean, 15 active+undersized+degraded, 69 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 730 op/s; 147/1752 objects degraded (8.390%) 2024-09-13T22:39:16.386 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:16 smithi105 ceph-mon[65554]: Health check failed: Degraded data redundancy: 147/1752 objects degraded (8.390%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:16.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 214262 9.38 MB/sec execute 464 sec latency 58.835 ms 2024-09-13T22:39:16.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:16 smithi118 ceph-mon[51566]: pgmap v189: 97 pgs: 4 active+undersized, 9 stale+active+clean, 15 active+undersized+degraded, 69 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 730 op/s; 147/1752 objects degraded (8.390%) 2024-09-13T22:39:16.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:16 smithi118 ceph-mon[51566]: Health check failed: Degraded data redundancy: 147/1752 objects degraded (8.390%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:16.597 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:16 smithi118 podman[56101]: 2024-09-13 22:39:16.367711444 +0000 UTC m=+0.689087908 container remove e26464d283938bdee19b00cbfff8ebb2c5f9226f0d78b0802ea46a4563bed91f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2) 2024-09-13T22:39:16.597 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:16 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-13T22:39:16.597 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:16 smithi118 systemd[1]: Stopped Ceph osd.4 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:39:16.597 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:16 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.4.service: Consumed 3min 32.480s CPU time. 2024-09-13T22:39:16.973 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:16 smithi118 systemd[1]: Starting Ceph osd.4 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:39:16.974 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:16 smithi118 podman[56196]: 2024-09-13 22:39:16.743968512 +0000 UTC m=+0.018769683 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:39:16.974 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:16 smithi118 podman[56196]: 2024-09-13 22:39:16.852303932 +0000 UTC m=+0.127105101 container create 6a2a432047a28b65b0d3fbd9850bd06c3c23a02b0cb2d4a7b39e0f94d9488d13 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-13T22:39:17.295 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:17 smithi118 podman[56196]: 2024-09-13 22:39:17.122154484 +0000 UTC m=+0.396955659 container init 6a2a432047a28b65b0d3fbd9850bd06c3c23a02b0cb2d4a7b39e0f94d9488d13 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T22:39:17.295 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:17 smithi118 podman[56196]: 2024-09-13 22:39:17.126439267 +0000 UTC m=+0.401240444 container start 6a2a432047a28b65b0d3fbd9850bd06c3c23a02b0cb2d4a7b39e0f94d9488d13 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate, ceph=True, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-13T22:39:17.295 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:17 smithi118 podman[56196]: 2024-09-13 22:39:17.222261166 +0000 UTC m=+0.497062337 container attach 6a2a432047a28b65b0d3fbd9850bd06c3c23a02b0cb2d4a7b39e0f94d9488d13 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:39:17.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 214576 9.37 MB/sec execute 465 sec latency 64.539 ms 2024-09-13T22:39:17.597 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:17 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate[56211]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:39:17.597 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:17 smithi118 bash[56196]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:39:17.597 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:17 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate[56211]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:39:17.597 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:17 smithi118 bash[56196]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:39:18.411 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:18 smithi105 ceph-mon[65554]: pgmap v190: 97 pgs: 5 active+undersized, 7 stale+active+clean, 22 active+undersized+degraded, 63 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 583 op/s; 211/1740 objects degraded (12.126%) 2024-09-13T22:39:18.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 214854 9.36 MB/sec execute 466 sec latency 33.429 ms 2024-09-13T22:39:18.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:18 smithi118 ceph-mon[51566]: pgmap v190: 97 pgs: 5 active+undersized, 7 stale+active+clean, 22 active+undersized+degraded, 63 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 583 op/s; 211/1740 objects degraded (12.126%) 2024-09-13T22:39:18.997 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:18 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate[56211]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T22:39:18.997 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:18 smithi118 bash[56196]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T22:39:18.997 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:18 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate[56211]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T22:39:18.997 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:18 smithi118 bash[56196]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T22:39:18.997 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:18 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate[56211]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T22:39:18.997 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:18 smithi118 bash[56196]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T22:39:19.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate[56211]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T22:39:19.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 bash[56196]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T22:39:19.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate[56211]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T22:39:19.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 bash[56196]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T22:39:19.348 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate[56211]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T22:39:19.348 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 bash[56196]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T22:39:19.348 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate[56211]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T22:39:19.348 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 bash[56196]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T22:39:19.348 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 podman[56196]: 2024-09-13 22:39:19.032421452 +0000 UTC m=+2.307222626 container died 6a2a432047a28b65b0d3fbd9850bd06c3c23a02b0cb2d4a7b39e0f94d9488d13 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T22:39:19.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 215306 9.37 MB/sec execute 467 sec latency 54.508 ms 2024-09-13T22:39:20.097 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 podman[56340]: 2024-09-13 22:39:19.777959249 +0000 UTC m=+0.738452141 container remove 6a2a432047a28b65b0d3fbd9850bd06c3c23a02b0cb2d4a7b39e0f94d9488d13 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:39:20.097 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:19 smithi118 podman[56396]: 2024-09-13 22:39:19.894331431 +0000 UTC m=+0.017861327 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:39:20.097 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:20 smithi118 podman[56396]: 2024-09-13 22:39:20.021546298 +0000 UTC m=+0.145076180 container create c8d592c38cddb153d150e7282080ae7d93e38a8b1fc891d1c862868fe631bf96 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T22:39:20.397 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:20 smithi118 podman[56396]: 2024-09-13 22:39:20.29996777 +0000 UTC m=+0.423497654 container init c8d592c38cddb153d150e7282080ae7d93e38a8b1fc891d1c862868fe631bf96 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T22:39:20.397 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:20 smithi118 podman[56396]: 2024-09-13 22:39:20.303592275 +0000 UTC m=+0.427122163 container start c8d592c38cddb153d150e7282080ae7d93e38a8b1fc891d1c862868fe631bf96 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T22:39:20.397 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:20 smithi118 ceph-osd[56414]: -- 172.21.15.118:0/3161707042 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b1172321e0 con 0x55b117208000 2024-09-13T22:39:20.484 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:20 smithi105 ceph-mon[65554]: pgmap v191: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 522 op/s; 271/1695 objects degraded (15.988%) 2024-09-13T22:39:20.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 215806 9.36 MB/sec execute 468 sec latency 41.317 ms 2024-09-13T22:39:20.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:20 smithi118 ceph-mon[51566]: pgmap v191: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 522 op/s; 271/1695 objects degraded (15.988%) 2024-09-13T22:39:20.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:20 smithi118 bash[56396]: c8d592c38cddb153d150e7282080ae7d93e38a8b1fc891d1c862868fe631bf96 2024-09-13T22:39:20.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:20 smithi118 systemd[1]: Started Ceph osd.4 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:39:21.496 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:21 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:21.496 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:21 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:21.496 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:21 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:39:21.496 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:21 smithi105 ceph-mon[65554]: pgmap v192: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 5.2 MiB/s wr, 585 op/s; 254/1644 objects degraded (15.450%) 2024-09-13T22:39:21.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 216117 9.37 MB/sec execute 469 sec latency 38.586 ms 2024-09-13T22:39:21.687 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:21 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:21.691 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:21 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:21.692 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:21 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:39:21.692 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:21 smithi118 ceph-mon[51566]: pgmap v192: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 5.2 MiB/s wr, 585 op/s; 254/1644 objects degraded (15.450%) 2024-09-13T22:39:22.061 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:21 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:39:21.685+0000 7f5df0782640 -1 Falling back to public interface 2024-09-13T22:39:22.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 216438 9.37 MB/sec execute 470 sec latency 78.024 ms 2024-09-13T22:39:22.598 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:22 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 254/1644 objects degraded (15.450%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:22.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:22 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 254/1644 objects degraded (15.450%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:23.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 216925 9.38 MB/sec execute 471 sec latency 44.576 ms 2024-09-13T22:39:23.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:23 smithi105 ceph-mon[65554]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.5 MiB/s wr, 556 op/s; 257/1656 objects degraded (15.519%) 2024-09-13T22:39:23.764 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:23 smithi118 ceph-mon[51566]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.5 MiB/s wr, 556 op/s; 257/1656 objects degraded (15.519%) 2024-09-13T22:39:24.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 217344 9.40 MB/sec execute 472 sec latency 34.726 ms 2024-09-13T22:39:25.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:25.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:25.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:25.429 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:25.541 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:25.541 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:25.541 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:25.542 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:25.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 217600 9.39 MB/sec execute 473 sec latency 61.207 ms 2024-09-13T22:39:26.455 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:26 smithi105 ceph-mon[65554]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 712 op/s; 271/1725 objects degraded (15.710%) 2024-09-13T22:39:26.545 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:26 smithi118 ceph-mon[51566]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 712 op/s; 271/1725 objects degraded (15.710%) 2024-09-13T22:39:26.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 217844 9.38 MB/sec execute 474 sec latency 88.801 ms 2024-09-13T22:39:27.485 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:27 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 272/1719 objects degraded (15.823%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:27.545 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:27 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 272/1719 objects degraded (15.823%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:27.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 218069 9.37 MB/sec execute 475 sec latency 73.962 ms 2024-09-13T22:39:27.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:27 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:39:27.575+0000 7f5df0782640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-13T22:39:28.503 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:28 smithi105 ceph-mon[65554]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 470 op/s; 272/1719 objects degraded (15.823%) 2024-09-13T22:39:28.558 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:28 smithi118 ceph-mon[51566]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 470 op/s; 272/1719 objects degraded (15.823%) 2024-09-13T22:39:28.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 218257 9.36 MB/sec execute 476 sec latency 55.054 ms 2024-09-13T22:39:28.705 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:39:28.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:28 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:39:28.803+0000 7f5df0782640 -1 osd.4 115 log_to_monitors true 2024-09-13T22:39:29.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 218571 9.35 MB/sec execute 477 sec latency 81.060 ms 2024-09-13T22:39:29.711 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:39:29.711 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (7m) 44s ago 16m 17.7M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (3m) 44s ago 16m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (3m) 5s ago 15m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (7m) 44s ago 16m 46.4M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (10m) 44s ago 10m 38.6M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (10m) 5s ago 10m 14.9M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (6m) 44s ago 17m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (5m) 5s ago 15m 441M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (5m) 44s ago 17m 50.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (4m) 5s ago 15m 41.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (10m) 44s ago 10m 279M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (10m) 5s ago 10m 81.8M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:39:29.712 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (7m) 44s ago 16m 8002k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (6m) 5s ago 15m 15.1M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (3m) 44s ago 14m 327M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (2m) 44s ago 14m 270M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (103s) 44s ago 14m 189M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (54s) 44s ago 13m 12.1M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (9s) 5s ago 13m 11.9M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 c8d592c38cdd 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (12m) 5s ago 12m 505M 2704M 17.2.0 e1d6a67b021e fb44ac67e459 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (12m) 5s ago 12m 430M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (11m) 5s ago 11m 700M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:39:29.713 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (5m) 44s ago 15m 61.0M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:39:29.776 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T22:39:29.777 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T22:39:29.778 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='osd.4 [v2:172.21.15.118:6800/2387728817,v1:172.21.15.118:6801/2387728817]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T22:39:29.778 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T22:39:29.778 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.1 MiB/s wr, 475 op/s; 270/1704 objects degraded (15.845%) 2024-09-13T22:39:29.778 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.778 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:39:29.778 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:29 smithi105 ceph-mon[65554]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:39:29.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:39:29.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:39:29.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:39:29.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='osd.4 [v2:172.21.15.118:6800/2387728817,v1:172.21.15.118:6801/2387728817]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.1 MiB/s wr, 475 op/s; 270/1704 objects degraded (15.845%) 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:39:29.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-mon[51566]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:39:30.304 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:39:30.304 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:39:30.304 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:39:30.304 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 4 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T22:39:30.305 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:39:30.306 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:39:30.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:39:29 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:39:29.939+0000 7f5de4525640 -1 osd.4 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:39:30.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 219008 9.36 MB/sec execute 478 sec latency 38.853 ms 2024-09-13T22:39:30.797 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:30 smithi105 ceph-mon[65554]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T22:39:30.797 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:30 smithi105 ceph-mon[65554]: osdmap e118: 8 total, 7 up, 8 in 2024-09-13T22:39:30.797 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:30 smithi105 ceph-mon[65554]: from='osd.4 [v2:172.21.15.118:6800/2387728817,v1:172.21.15.118:6801/2387728817]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:39:30.797 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:30 smithi105 ceph-mon[65554]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:39:30.797 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:30 smithi105 ceph-mon[65554]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:39:30.797 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:30 smithi105 ceph-mon[65554]: from='osd.4 ' entity='osd.4' 2024-09-13T22:39:30.798 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:30 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/347719067' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "11/23 daemons upgraded", 2024-09-13T22:39:30.822 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:39:30.823 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:39:30.823 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:39:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:30 smithi118 ceph-mon[51566]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T22:39:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:30 smithi118 ceph-mon[51566]: osdmap e118: 8 total, 7 up, 8 in 2024-09-13T22:39:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:30 smithi118 ceph-mon[51566]: from='osd.4 [v2:172.21.15.118:6800/2387728817,v1:172.21.15.118:6801/2387728817]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:39:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:30 smithi118 ceph-mon[51566]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:39:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:30 smithi118 ceph-mon[51566]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:39:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:30 smithi118 ceph-mon[51566]: from='osd.4 ' entity='osd.4' 2024-09-13T22:39:31.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:30 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/347719067' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:39:31.284 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_WARN Degraded data redundancy: 270/1704 objects degraded (15.845%), 28 pgs degraded 2024-09-13T22:39:31.284 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 270/1704 objects degraded (15.845%), 28 pgs degraded 2024-09-13T22:39:31.284 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-13T22:39:31.284 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1f is active+undersized+degraded, acting [0,2] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-13T22:39:31.285 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T22:39:31.286 INFO:teuthology.orchestra.run.smithi105.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-13T22:39:31.595 INFO:teuthology.orchestra.run.smithi105.stdout: 5 219210 9.34 MB/sec execute 479 sec latency 653.526 ms 2024-09-13T22:39:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:31 smithi118 ceph-mon[51566]: from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:39:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:31 smithi118 ceph-mon[51566]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 451 op/s; 259/1614 objects degraded (16.047%) 2024-09-13T22:39:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:31 smithi118 ceph-mon[51566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:39:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:31 smithi118 ceph-mon[51566]: osd.4 [v2:172.21.15.118:6800/2387728817,v1:172.21.15.118:6801/2387728817] boot 2024-09-13T22:39:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:31 smithi118 ceph-mon[51566]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T22:39:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:31 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:39:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:31 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/1894875382' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:39:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:31 smithi105 ceph-mon[65554]: from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:39:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:31 smithi105 ceph-mon[65554]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 451 op/s; 259/1614 objects degraded (16.047%) 2024-09-13T22:39:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:31 smithi105 ceph-mon[65554]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:39:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:31 smithi105 ceph-mon[65554]: osd.4 [v2:172.21.15.118:6800/2387728817,v1:172.21.15.118:6801/2387728817] boot 2024-09-13T22:39:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:31 smithi105 ceph-mon[65554]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T22:39:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:31 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T22:39:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:31 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/1894875382' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:39:32.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 219210 9.33 MB/sec execute 480 sec latency 1653.576 ms 2024-09-13T22:39:33.214 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:32 smithi105 ceph-mon[65554]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T22:39:33.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:32 smithi118 ceph-mon[51566]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T22:39:33.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 219479 9.32 MB/sec execute 481 sec latency 2094.995 ms 2024-09-13T22:39:33.971 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:33 smithi105 ceph-mon[65554]: pgmap v201: 97 pgs: 4 remapped+peering, 6 peering, 9 active+undersized, 18 active+undersized+degraded, 60 active+clean; 351 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.1 MiB/s wr, 445 op/s; 155/1626 objects degraded (9.533%) 2024-09-13T22:39:33.971 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:33 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 155/1626 objects degraded (9.533%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:33.971 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:33 smithi105 ceph-mon[65554]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T22:39:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:33 smithi118 ceph-mon[51566]: pgmap v201: 97 pgs: 4 remapped+peering, 6 peering, 9 active+undersized, 18 active+undersized+degraded, 60 active+clean; 351 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.1 MiB/s wr, 445 op/s; 155/1626 objects degraded (9.533%) 2024-09-13T22:39:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:33 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 155/1626 objects degraded (9.533%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:33 smithi118 ceph-mon[51566]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T22:39:34.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 219782 9.32 MB/sec execute 482 sec latency 625.805 ms 2024-09-13T22:39:35.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:34 smithi105 ceph-mon[65554]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T22:39:35.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:34 smithi118 ceph-mon[51566]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T22:39:35.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 220091 9.31 MB/sec execute 483 sec latency 1016.609 ms 2024-09-13T22:39:36.223 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:35 smithi105 ceph-mon[65554]: pgmap v204: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 3 remapped+peering, 12 peering, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.4 MiB/s wr, 466 op/s; 30/1620 objects degraded (1.852%); 2.0 MiB/s, 46 objects/s recovering 2024-09-13T22:39:36.223 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:35 smithi105 ceph-mon[65554]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T22:39:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:35 smithi118 ceph-mon[51566]: pgmap v204: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 3 remapped+peering, 12 peering, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.4 MiB/s wr, 466 op/s; 30/1620 objects degraded (1.852%); 2.0 MiB/s, 46 objects/s recovering 2024-09-13T22:39:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:35 smithi118 ceph-mon[51566]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T22:39:36.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 220397 9.31 MB/sec execute 484 sec latency 896.247 ms 2024-09-13T22:39:37.012 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:36 smithi105 ceph-mon[65554]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T22:39:37.012 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:36 smithi105 ceph-mon[65554]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T22:39:37.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:36 smithi118 ceph-mon[51566]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T22:39:37.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:36 smithi118 ceph-mon[51566]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T22:39:37.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 220848 9.33 MB/sec execute 485 sec latency 792.669 ms 2024-09-13T22:39:38.023 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:37 smithi105 ceph-mon[65554]: pgmap v207: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 3 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s wr, 197 op/s; 30/1617 objects degraded (1.855%); 2.3 MiB/s, 59 objects/s recovering 2024-09-13T22:39:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:37 smithi118 ceph-mon[51566]: pgmap v207: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 3 remapped+peering, 6 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s wr, 197 op/s; 30/1617 objects degraded (1.855%); 2.3 MiB/s, 59 objects/s recovering 2024-09-13T22:39:38.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 221195 9.33 MB/sec execute 486 sec latency 39.582 ms 2024-09-13T22:39:39.047 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:39 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 30/1596 objects degraded (1.880%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:39.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:39 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 30/1596 objects degraded (1.880%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:39.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 221568 9.34 MB/sec execute 487 sec latency 37.831 ms 2024-09-13T22:39:40.067 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:40 smithi105 ceph-mon[65554]: pgmap v209: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.8 MiB/s wr, 277 op/s; 30/1596 objects degraded (1.880%); 2.6 MiB/s, 47 keys/s, 59 objects/s recovering 2024-09-13T22:39:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:40 smithi118 ceph-mon[51566]: pgmap v209: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.8 MiB/s wr, 277 op/s; 30/1596 objects degraded (1.880%); 2.6 MiB/s, 47 keys/s, 59 objects/s recovering 2024-09-13T22:39:40.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 221886 9.34 MB/sec execute 488 sec latency 52.804 ms 2024-09-13T22:39:41.099 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:41 smithi105 ceph-mon[65554]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1596 objects degraded (1.880%), 2 pgs degraded) 2024-09-13T22:39:41.099 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:41 smithi105 ceph-mon[65554]: Cluster is now healthy 2024-09-13T22:39:41.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:41 smithi118 ceph-mon[51566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1596 objects degraded (1.880%), 2 pgs degraded) 2024-09-13T22:39:41.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:41 smithi118 ceph-mon[51566]: Cluster is now healthy 2024-09-13T22:39:41.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 222102 9.32 MB/sec execute 489 sec latency 92.997 ms 2024-09-13T22:39:42.154 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:42 smithi105 ceph-mon[65554]: pgmap v210: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.8 MiB/s wr, 339 op/s; 866 KiB/s, 39 keys/s, 29 objects/s recovering 2024-09-13T22:39:42.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:42 smithi118 ceph-mon[51566]: pgmap v210: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.8 MiB/s wr, 339 op/s; 866 KiB/s, 39 keys/s, 29 objects/s recovering 2024-09-13T22:39:42.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 222267 9.32 MB/sec execute 490 sec latency 73.478 ms 2024-09-13T22:39:43.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 222471 9.31 MB/sec execute 491 sec latency 72.460 ms 2024-09-13T22:39:44.178 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:44 smithi105 ceph-mon[65554]: pgmap v211: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 428 op/s; 656 KiB/s, 29 keys/s, 22 objects/s recovering 2024-09-13T22:39:44.178 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T22:39:44.178 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:44.178 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:39:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:44 smithi118 ceph-mon[51566]: pgmap v211: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 428 op/s; 656 KiB/s, 29 keys/s, 22 objects/s recovering 2024-09-13T22:39:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T22:39:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:44.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:39:44.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 222697 9.29 MB/sec execute 492 sec latency 68.474 ms 2024-09-13T22:39:45.211 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:45 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T22:39:45.211 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:45 smithi105 ceph-mon[65554]: Upgrade: osd.5 is safe to restart 2024-09-13T22:39:45.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:45 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T22:39:45.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:45 smithi118 ceph-mon[51566]: Upgrade: osd.5 is safe to restart 2024-09-13T22:39:45.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 223125 9.29 MB/sec execute 493 sec latency 55.753 ms 2024-09-13T22:39:46.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:46 smithi105 ceph-mon[65554]: pgmap v212: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 528 op/s; 583 KiB/s, 26 keys/s, 19 objects/s recovering 2024-09-13T22:39:46.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:46 smithi105 ceph-mon[65554]: Upgrade: Updating osd.5 2024-09-13T22:39:46.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:46.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T22:39:46.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:39:46.470 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:46 smithi105 ceph-mon[65554]: Deploying daemon osd.5 on smithi118 2024-09-13T22:39:46.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 223537 9.29 MB/sec execute 494 sec latency 44.204 ms 2024-09-13T22:39:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:46 smithi118 ceph-mon[51566]: pgmap v212: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 528 op/s; 583 KiB/s, 26 keys/s, 19 objects/s recovering 2024-09-13T22:39:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:46 smithi118 ceph-mon[51566]: Upgrade: Updating osd.5 2024-09-13T22:39:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T22:39:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:39:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:46 smithi118 ceph-mon[51566]: Deploying daemon osd.5 on smithi118 2024-09-13T22:39:47.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 223924 9.30 MB/sec execute 495 sec latency 49.655 ms 2024-09-13T22:39:48.499 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:48 smithi105 ceph-mon[65554]: pgmap v213: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 479 op/s; 408 KiB/s, 23 keys/s, 12 objects/s recovering 2024-09-13T22:39:48.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 224363 9.31 MB/sec execute 496 sec latency 45.112 ms 2024-09-13T22:39:48.722 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:48 smithi118 ceph-mon[51566]: pgmap v213: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 479 op/s; 408 KiB/s, 23 keys/s, 12 objects/s recovering 2024-09-13T22:39:48.981 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:48 smithi118 systemd[1]: Stopping Ceph osd.5 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:39:49.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:49 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[32865]: 2024-09-13T22:39:49.011+0000 7f6f54223700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:39:49.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:49 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[32865]: 2024-09-13T22:39:49.011+0000 7f6f54223700 -1 osd.5 125 *** Got signal Terminated *** 2024-09-13T22:39:49.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:49 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[32865]: 2024-09-13T22:39:49.011+0000 7f6f54223700 -1 osd.5 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:39:49.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 224679 9.32 MB/sec execute 497 sec latency 57.227 ms 2024-09-13T22:39:50.041 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:49 smithi105 ceph-mon[65554]: pgmap v214: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.6 MiB/s wr, 496 op/s; 342 KiB/s, 19 keys/s, 10 objects/s recovering 2024-09-13T22:39:50.041 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:50.041 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:49 smithi105 ceph-mon[65554]: osd.5 marked itself down and dead 2024-09-13T22:39:50.248 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:49 smithi118 ceph-mon[51566]: pgmap v214: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.6 MiB/s wr, 496 op/s; 342 KiB/s, 19 keys/s, 10 objects/s recovering 2024-09-13T22:39:50.249 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:50.249 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:49 smithi118 ceph-mon[51566]: osd.5 marked itself down and dead 2024-09-13T22:39:50.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 224880 9.31 MB/sec execute 498 sec latency 618.304 ms 2024-09-13T22:39:50.882 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:50 smithi118 podman[58211]: 2024-09-13 22:39:50.600340303 +0000 UTC m=+1.704201651 container died fb44ac67e459de0c37010f07f2014553a0366ca04605a31f8f219a51161ffa4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, release=754) 2024-09-13T22:39:51.190 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:50 smithi118 ceph-mon[51566]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:39:51.190 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:50 smithi118 ceph-mon[51566]: osdmap e126: 8 total, 7 up, 8 in 2024-09-13T22:39:51.191 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:50 smithi118 podman[58211]: 2024-09-13 22:39:50.971708905 +0000 UTC m=+2.075570261 container cleanup fb44ac67e459de0c37010f07f2014553a0366ca04605a31f8f219a51161ffa4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-type=git, distribution-scope=public, version=8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_CLEAN=True, GIT_BRANCH=HEAD, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64) 2024-09-13T22:39:51.191 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:50 smithi118 bash[58211]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5 2024-09-13T22:39:51.198 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:50 smithi105 ceph-mon[65554]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:39:51.198 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:50 smithi105 ceph-mon[65554]: osdmap e126: 8 total, 7 up, 8 in 2024-09-13T22:39:51.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 225038 9.30 MB/sec execute 499 sec latency 1010.158 ms 2024-09-13T22:39:51.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:51 smithi118 podman[58237]: 2024-09-13 22:39:51.190310823 +0000 UTC m=+0.592456008 container remove fb44ac67e459de0c37010f07f2014553a0366ca04605a31f8f219a51161ffa4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, RELEASE=HEAD, io.openshift.expose-services=, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , release=754, version=8, ceph=True, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T22:39:51.903 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:51 smithi118 podman[58324]: 2024-09-13 22:39:51.544626029 +0000 UTC m=+0.016592789 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:39:51.904 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:51 smithi118 podman[58324]: 2024-09-13 22:39:51.675018026 +0000 UTC m=+0.146984774 container create 10778f582f6582bb06dd15e6c84b395eb2a99f8f87efd7189a1068460e85ef8c (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T22:39:51.904 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:51 smithi118 podman[58324]: 2024-09-13 22:39:51.903041475 +0000 UTC m=+0.375008232 container init 10778f582f6582bb06dd15e6c84b395eb2a99f8f87efd7189a1068460e85ef8c (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:39:51.972 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:51 smithi105 ceph-mon[65554]: pgmap v216: 97 pgs: 13 stale+active+clean, 84 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 451 op/s 2024-09-13T22:39:51.972 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:51 smithi105 ceph-mon[65554]: osdmap e127: 8 total, 7 up, 8 in 2024-09-13T22:39:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:51 smithi118 ceph-mon[51566]: pgmap v216: 97 pgs: 13 stale+active+clean, 84 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 451 op/s 2024-09-13T22:39:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:51 smithi118 ceph-mon[51566]: osdmap e127: 8 total, 7 up, 8 in 2024-09-13T22:39:52.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:51 smithi118 podman[58324]: 2024-09-13 22:39:51.907238476 +0000 UTC m=+0.379205236 container start 10778f582f6582bb06dd15e6c84b395eb2a99f8f87efd7189a1068460e85ef8c (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T22:39:52.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:52 smithi118 podman[58324]: 2024-09-13 22:39:52.003178859 +0000 UTC m=+0.475145614 container attach 10778f582f6582bb06dd15e6c84b395eb2a99f8f87efd7189a1068460e85ef8c (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T22:39:52.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:52 smithi118 podman[58324]: 2024-09-13 22:39:52.105310931 +0000 UTC m=+0.577277680 container died 10778f582f6582bb06dd15e6c84b395eb2a99f8f87efd7189a1068460e85ef8c (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T22:39:52.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 225340 9.30 MB/sec execute 500 sec latency 63.398 ms 2024-09-13T22:39:53.013 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:52 smithi118 podman[58348]: 2024-09-13 22:39:52.750652484 +0000 UTC m=+0.638772692 container remove 10778f582f6582bb06dd15e6c84b395eb2a99f8f87efd7189a1068460e85ef8c (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T22:39:53.013 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:52 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-13T22:39:53.013 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:52 smithi118 systemd[1]: Stopped Ceph osd.5 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:39:53.013 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:52 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 15.433s CPU time. 2024-09-13T22:39:53.013 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:52 smithi118 systemd[1]: Starting Ceph osd.5 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:39:53.236 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:53 smithi105 ceph-mon[65554]: Health check failed: Degraded data redundancy: 100/1716 objects degraded (5.828%), 12 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:53.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:53 smithi118 ceph-mon[51566]: Health check failed: Degraded data redundancy: 100/1716 objects degraded (5.828%), 12 pgs degraded (PG_DEGRADED) 2024-09-13T22:39:53.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:53 smithi118 podman[58444]: 2024-09-13 22:39:53.127901885 +0000 UTC m=+0.021443941 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:39:53.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:53 smithi118 podman[58444]: 2024-09-13 22:39:53.26064418 +0000 UTC m=+0.154186225 container create 93828675422f9516a2fb2fdad9eaa3d6bc2002301e6b806c1b961d83480f61f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T22:39:53.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 225731 9.30 MB/sec execute 501 sec latency 48.079 ms 2024-09-13T22:39:53.648 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:53 smithi118 podman[58444]: 2024-09-13 22:39:53.546933282 +0000 UTC m=+0.440475332 container init 93828675422f9516a2fb2fdad9eaa3d6bc2002301e6b806c1b961d83480f61f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:39:53.648 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:53 smithi118 podman[58444]: 2024-09-13 22:39:53.550836953 +0000 UTC m=+0.444379008 container start 93828675422f9516a2fb2fdad9eaa3d6bc2002301e6b806c1b961d83480f61f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T22:39:53.648 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:53 smithi118 podman[58444]: 2024-09-13 22:39:53.647360787 +0000 UTC m=+0.540902838 container attach 93828675422f9516a2fb2fdad9eaa3d6bc2002301e6b806c1b961d83480f61f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-13T22:39:54.013 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:53 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate[58459]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:39:54.013 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:53 smithi118 bash[58444]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:39:54.013 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:53 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate[58459]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:39:54.013 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:53 smithi118 bash[58444]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:39:54.260 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:54 smithi105 ceph-mon[65554]: pgmap v218: 97 pgs: 3 active+undersized, 8 stale+active+clean, 12 active+undersized+degraded, 74 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.6 MiB/s wr, 447 op/s; 100/1716 objects degraded (5.828%) 2024-09-13T22:39:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:54 smithi118 ceph-mon[51566]: pgmap v218: 97 pgs: 3 active+undersized, 8 stale+active+clean, 12 active+undersized+degraded, 74 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.6 MiB/s wr, 447 op/s; 100/1716 objects degraded (5.828%) 2024-09-13T22:39:54.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 226095 9.29 MB/sec execute 502 sec latency 48.086 ms 2024-09-13T22:39:55.096 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:54 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate[58459]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T22:39:55.096 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:54 smithi118 bash[58444]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T22:39:55.097 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:54 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate[58459]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T22:39:55.097 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:54 smithi118 bash[58444]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T22:39:55.097 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate[58459]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T22:39:55.097 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 bash[58444]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T22:39:55.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate[58459]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T22:39:55.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 bash[58444]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T22:39:55.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate[58459]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T22:39:55.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 bash[58444]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T22:39:55.348 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate[58459]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T22:39:55.348 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 bash[58444]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T22:39:55.348 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate[58459]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T22:39:55.348 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 bash[58444]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T22:39:55.348 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 podman[58444]: 2024-09-13 22:39:55.128438467 +0000 UTC m=+2.021980510 container died 93828675422f9516a2fb2fdad9eaa3d6bc2002301e6b806c1b961d83480f61f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T22:39:55.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 226453 9.29 MB/sec execute 503 sec latency 43.491 ms 2024-09-13T22:39:56.106 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:56 smithi105 ceph-mon[65554]: pgmap v219: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 463 op/s; 188/1677 objects degraded (11.210%) 2024-09-13T22:39:56.113 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:56 smithi118 ceph-mon[51566]: pgmap v219: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 463 op/s; 188/1677 objects degraded (11.210%) 2024-09-13T22:39:56.113 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:55 smithi118 podman[58590]: 2024-09-13 22:39:55.862097539 +0000 UTC m=+0.727595218 container remove 93828675422f9516a2fb2fdad9eaa3d6bc2002301e6b806c1b961d83480f61f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T22:39:56.113 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:56 smithi118 podman[58650]: 2024-09-13 22:39:55.983776562 +0000 UTC m=+0.019258884 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:39:56.366 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:56 smithi118 podman[58650]: 2024-09-13 22:39:56.113452008 +0000 UTC m=+0.148934314 container create f03e887cfb9eb0fc1d3c91b69a5eac729960f4ef90624281644d74701f1fb6e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:39:56.367 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:56 smithi118 podman[58650]: 2024-09-13 22:39:56.36659309 +0000 UTC m=+0.402075406 container init f03e887cfb9eb0fc1d3c91b69a5eac729960f4ef90624281644d74701f1fb6e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:39:56.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 226829 9.29 MB/sec execute 504 sec latency 45.055 ms 2024-09-13T22:39:56.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:56 smithi118 podman[58650]: 2024-09-13 22:39:56.370146934 +0000 UTC m=+0.405629244 container start f03e887cfb9eb0fc1d3c91b69a5eac729960f4ef90624281644d74701f1fb6e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T22:39:56.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:56 smithi118 ceph-osd[58668]: -- 172.21.15.118:0/225580766 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563a2a1be1e0 con 0x563a293afc00 2024-09-13T22:39:56.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:56 smithi118 bash[58650]: f03e887cfb9eb0fc1d3c91b69a5eac729960f4ef90624281644d74701f1fb6e4 2024-09-13T22:39:56.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:56 smithi118 systemd[1]: Started Ceph osd.5 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:39:57.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 227206 9.29 MB/sec execute 505 sec latency 38.109 ms 2024-09-13T22:39:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:57 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:39:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:57 smithi118 ceph-mon[51566]: pgmap v220: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.4 MiB/s wr, 345 op/s; 183/1641 objects degraded (11.152%) 2024-09-13T22:39:57.638 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:57.638 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:57.638 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:57 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:39:57.638 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:57 smithi105 ceph-mon[65554]: pgmap v220: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.4 MiB/s wr, 345 op/s; 183/1641 objects degraded (11.152%) 2024-09-13T22:39:58.443 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:39:58 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:39:58.258+0000 7fb4db73b640 -1 Falling back to public interface 2024-09-13T22:39:58.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 227567 9.30 MB/sec execute 506 sec latency 48.048 ms 2024-09-13T22:39:59.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 228045 9.31 MB/sec execute 507 sec latency 31.232 ms 2024-09-13T22:39:59.940 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:39:59.940 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:39:59.940 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:59 smithi105 ceph-mon[65554]: pgmap v221: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.0 MiB/s wr, 498 op/s; 183/1641 objects degraded (11.152%) 2024-09-13T22:39:59.940 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:39:59 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 183/1641 objects degraded (11.152%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:00.140 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:00.140 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:40:00.140 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:59 smithi118 ceph-mon[51566]: pgmap v221: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.0 MiB/s wr, 498 op/s; 183/1641 objects degraded (11.152%) 2024-09-13T22:40:00.140 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:39:59 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 183/1641 objects degraded (11.152%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:00.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 228357 9.31 MB/sec execute 508 sec latency 48.077 ms 2024-09-13T22:40:00.962 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 183/1641 objects degraded (11.152%), 23 pgs degraded 2024-09-13T22:40:00.962 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T22:40:00.962 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: osd.5 (root=default,host=smithi118) is down 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: [WRN] PG_DEGRADED: Degraded data redundancy: 183/1641 objects degraded (11.152%), 23 pgs degraded 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-13T22:40:00.963 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:00.964 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:01.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 183/1641 objects degraded (11.152%), 23 pgs degraded 2024-09-13T22:40:01.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T22:40:01.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: osd.5 (root=default,host=smithi118) is down 2024-09-13T22:40:01.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: [WRN] PG_DEGRADED: Degraded data redundancy: 183/1641 objects degraded (11.152%), 23 pgs degraded 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-13T22:40:01.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:01.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:01.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 228635 9.31 MB/sec execute 509 sec latency 57.259 ms 2024-09-13T22:40:01.740 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:40:02.505 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:02 smithi105 ceph-mon[65554]: pgmap v222: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 545 op/s; 180/1650 objects degraded (10.909%) 2024-09-13T22:40:02.505 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:02 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:02.505 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:02 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:02.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 228827 9.30 MB/sec execute 510 sec latency 71.396 ms 2024-09-13T22:40:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:02 smithi118 ceph-mon[51566]: pgmap v222: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 545 op/s; 180/1650 objects degraded (10.909%) 2024-09-13T22:40:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:02 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:02 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (7m) 77s ago 17m 17.7M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (4m) 77s ago 16m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (4m) 2s ago 16m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (7m) 77s ago 16m 46.4M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (11m) 77s ago 11m 38.6M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (10m) 2s ago 10m 15.0M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (6m) 77s ago 18m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (6m) 2s ago 16m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:40:02.728 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (5m) 77s ago 18m 50.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (5m) 2s ago 16m 46.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (10m) 77s ago 10m 279M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (10m) 2s ago 10m 81.9M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (7m) 77s ago 16m 8002k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (7m) 2s ago 16m 15.2M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (3m) 77s ago 15m 327M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (3m) 77s ago 15m 270M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (2m) 77s ago 14m 189M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (87s) 77s ago 14m 12.1M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (42s) 2s ago 13m 271M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 c8d592c38cdd 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (6s) 2s ago 13m 14.2M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 f03e887cfb9e 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (12m) 2s ago 12m 468M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (12m) 2s ago 12m 750M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:40:02.729 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (6m) 77s ago 15m 61.0M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:40:03.251 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:40:03.251 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:40:03.251 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:40:03.251 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:40:03.251 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:40:03.251 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:40:03.251 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 5 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 9 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:40:03.252 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:40:03.515 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:03 smithi105 ceph-mon[65554]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:03.515 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:03 smithi105 ceph-mon[65554]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:03.515 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:03 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2958494480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:03.596 INFO:teuthology.orchestra.run.smithi105.stdout: 5 229058 9.29 MB/sec execute 511 sec latency 109.940 ms 2024-09-13T22:40:03.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:03 smithi118 ceph-mon[51566]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:03.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:03 smithi118 ceph-mon[51566]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:03.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:03 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2958494480' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:03.703 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "12/23 daemons upgraded", 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:40:03.704 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:40:04.224 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 187/1692 objects degraded (11.052%), 23 pgs degraded 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: osd.5 (root=default,host=smithi118) is down 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 187/1692 objects degraded (11.052%), 23 pgs degraded 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-13T22:40:04.225 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-13T22:40:04.226 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-13T22:40:04.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:04 smithi105 ceph-mon[65554]: from='client.44297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:04.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:04 smithi105 ceph-mon[65554]: pgmap v223: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.2 MiB/s wr, 529 op/s; 187/1692 objects degraded (11.052%) 2024-09-13T22:40:04.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:04 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 187/1692 objects degraded (11.052%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:04.530 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:04 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/715889937' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:40:04.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:40:04 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:40:04.446+0000 7fb4db73b640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-13T22:40:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:04 smithi118 ceph-mon[51566]: from='client.44297 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:04 smithi118 ceph-mon[51566]: pgmap v223: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.2 MiB/s wr, 529 op/s; 187/1692 objects degraded (11.052%) 2024-09-13T22:40:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:04 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 187/1692 objects degraded (11.052%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:04.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:04 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/715889937' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:40:04.605 INFO:teuthology.orchestra.run.smithi105.stdout: 5 229346 9.29 MB/sec execute 512 sec latency 48.546 ms 2024-09-13T22:40:05.571 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:05 smithi105 ceph-mon[65554]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:05.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:05 smithi118 ceph-mon[51566]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:05.605 INFO:teuthology.orchestra.run.smithi105.stdout: 5 229694 9.28 MB/sec execute 513 sec latency 59.735 ms 2024-09-13T22:40:06.035 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:40:05 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:40:05.648+0000 7fb4db73b640 -1 osd.5 125 log_to_monitors true 2024-09-13T22:40:06.595 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:06 smithi105 ceph-mon[65554]: pgmap v224: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.1 MiB/s wr, 522 op/s; 181/1665 objects degraded (10.871%) 2024-09-13T22:40:06.595 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:06 smithi105 ceph-mon[65554]: from='osd.5 [v2:172.21.15.118:6808/1184472985,v1:172.21.15.118:6809/1184472985]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T22:40:06.595 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:06 smithi105 ceph-mon[65554]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T22:40:06.605 INFO:teuthology.orchestra.run.smithi105.stdout: 5 230188 9.29 MB/sec execute 514 sec latency 53.001 ms 2024-09-13T22:40:06.650 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:06 smithi118 ceph-mon[51566]: pgmap v224: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.1 MiB/s wr, 522 op/s; 181/1665 objects degraded (10.871%) 2024-09-13T22:40:06.650 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:06 smithi118 ceph-mon[51566]: from='osd.5 [v2:172.21.15.118:6808/1184472985,v1:172.21.15.118:6809/1184472985]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T22:40:06.650 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:06 smithi118 ceph-mon[51566]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T22:40:07.097 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:40:06 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:40:06.648+0000 7fb4cf4de640 -1 osd.5 125 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:40:07.605 INFO:teuthology.orchestra.run.smithi105.stdout: 5 230590 9.28 MB/sec execute 515 sec latency 48.688 ms 2024-09-13T22:40:07.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T22:40:07.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='osd.5 [v2:172.21.15.118:6808/1184472985,v1:172.21.15.118:6809/1184472985]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:40:07.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: osdmap e128: 8 total, 7 up, 8 in 2024-09-13T22:40:07.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:40:07.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='osd.5 ' entity='osd.5' 2024-09-13T22:40:07.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:07.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:07.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:40:07.607 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:40:07.608 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:07.608 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:07.608 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:40:07.608 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:07.608 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:07.608 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:07.608 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T22:40:07.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T22:40:07.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='osd.5 [v2:172.21.15.118:6808/1184472985,v1:172.21.15.118:6809/1184472985]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: osdmap e128: 8 total, 7 up, 8 in 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='osd.5 ' entity='osd.5' 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:07.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:07.849 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T22:40:08.605 INFO:teuthology.orchestra.run.smithi105.stdout: 5 230613 9.27 MB/sec execute 516 sec latency 949.626 ms 2024-09-13T22:40:08.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:08 smithi105 ceph-mon[65554]: pgmap v226: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 595 op/s; 180/1668 objects degraded (10.791%) 2024-09-13T22:40:08.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:08 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T22:40:08.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:08 smithi105 ceph-mon[65554]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-13T22:40:08.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:08 smithi105 ceph-mon[65554]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:40:08.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:08 smithi105 ceph-mon[65554]: osd.5 [v2:172.21.15.118:6808/1184472985,v1:172.21.15.118:6809/1184472985] boot 2024-09-13T22:40:08.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:08 smithi105 ceph-mon[65554]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T22:40:08.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:08 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:40:08.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:08 smithi118 ceph-mon[51566]: pgmap v226: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 595 op/s; 180/1668 objects degraded (10.791%) 2024-09-13T22:40:08.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:08 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T22:40:08.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:08 smithi118 ceph-mon[51566]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-13T22:40:08.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:08 smithi118 ceph-mon[51566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:40:08.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:08 smithi118 ceph-mon[51566]: osd.5 [v2:172.21.15.118:6808/1184472985,v1:172.21.15.118:6809/1184472985] boot 2024-09-13T22:40:08.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:08 smithi118 ceph-mon[51566]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T22:40:08.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:08 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T22:40:09.605 INFO:teuthology.orchestra.run.smithi105.stdout: 5 230613 9.25 MB/sec execute 517 sec latency 1949.682 ms 2024-09-13T22:40:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:09 smithi118 ceph-mon[51566]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T22:40:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:09 smithi118 ceph-mon[51566]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.4 MiB/s wr, 359 op/s; 176/1608 objects degraded (10.945%) 2024-09-13T22:40:10.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:09 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 180/1668 objects degraded (10.791%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:10.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:09 smithi105 ceph-mon[65554]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T22:40:10.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:09 smithi105 ceph-mon[65554]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.4 MiB/s wr, 359 op/s; 176/1608 objects degraded (10.945%) 2024-09-13T22:40:10.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:09 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 180/1668 objects degraded (10.791%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:10.605 INFO:teuthology.orchestra.run.smithi105.stdout: 5 230698 9.23 MB/sec execute 518 sec latency 2659.194 ms 2024-09-13T22:40:10.857 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:10 smithi105 ceph-mon[65554]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T22:40:11.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:10 smithi118 ceph-mon[51566]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T22:40:11.605 INFO:teuthology.orchestra.run.smithi105.stdout: 5 231080 9.25 MB/sec execute 519 sec latency 66.430 ms 2024-09-13T22:40:11.880 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:11 smithi105 ceph-mon[65554]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T22:40:11.880 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:11 smithi105 ceph-mon[65554]: pgmap v232: 97 pgs: 4 activating+undersized+degraded+remapped, 6 active+undersized, 16 active+undersized+degraded, 71 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.5 MiB/s wr, 417 op/s; 191/1593 objects degraded (11.990%); 22 B/s, 0 objects/s recovering 2024-09-13T22:40:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:11 smithi118 ceph-mon[51566]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T22:40:12.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:11 smithi118 ceph-mon[51566]: pgmap v232: 97 pgs: 4 activating+undersized+degraded+remapped, 6 active+undersized, 16 active+undersized+degraded, 71 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.5 MiB/s wr, 417 op/s; 191/1593 objects degraded (11.990%); 22 B/s, 0 objects/s recovering 2024-09-13T22:40:12.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 231490 9.25 MB/sec execute 520 sec latency 626.218 ms 2024-09-13T22:40:12.916 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:12 smithi105 ceph-mon[65554]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T22:40:13.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:12 smithi118 ceph-mon[51566]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T22:40:13.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 231769 9.26 MB/sec execute 521 sec latency 848.871 ms 2024-09-13T22:40:13.927 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:13 smithi105 ceph-mon[65554]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T22:40:13.927 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:13 smithi105 ceph-mon[65554]: pgmap v235: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 4 activating+undersized+degraded+remapped, 90 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 550 op/s; 128/1599 objects degraded (8.005%); 183 KiB/s, 30 objects/s recovering 2024-09-13T22:40:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:13 smithi118 ceph-mon[51566]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T22:40:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:13 smithi118 ceph-mon[51566]: pgmap v235: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 4 activating+undersized+degraded+remapped, 90 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 550 op/s; 128/1599 objects degraded (8.005%); 183 KiB/s, 30 objects/s recovering 2024-09-13T22:40:14.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 232095 9.25 MB/sec execute 522 sec latency 968.757 ms 2024-09-13T22:40:14.941 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:14 smithi105 ceph-mon[65554]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T22:40:14.941 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:14.941 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:40:14.941 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:14 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 128/1599 objects degraded (8.005%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:14 smithi118 ceph-mon[51566]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T22:40:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:40:15.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:14 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 128/1599 objects degraded (8.005%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:15.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 232316 9.24 MB/sec execute 523 sec latency 968.799 ms 2024-09-13T22:40:15.980 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:15 smithi105 ceph-mon[65554]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T22:40:15.980 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:15 smithi105 ceph-mon[65554]: pgmap v238: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 93 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 300 op/s; 86/1638 objects degraded (5.250%); 1.2 MiB/s, 40 objects/s recovering 2024-09-13T22:40:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:15 smithi118 ceph-mon[51566]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T22:40:16.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:15 smithi118 ceph-mon[51566]: pgmap v238: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 activating+undersized+degraded+remapped, 93 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 300 op/s; 86/1638 objects degraded (5.250%); 1.2 MiB/s, 40 objects/s recovering 2024-09-13T22:40:16.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 232613 9.24 MB/sec execute 524 sec latency 40.273 ms 2024-09-13T22:40:17.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 232956 9.24 MB/sec execute 525 sec latency 60.295 ms 2024-09-13T22:40:18.010 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:17 smithi105 ceph-mon[65554]: pgmap v239: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 261 op/s; 36/1638 objects degraded (2.198%); 1.8 MiB/s, 26 objects/s recovering 2024-09-13T22:40:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:17 smithi118 ceph-mon[51566]: pgmap v239: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 261 op/s; 36/1638 objects degraded (2.198%); 1.8 MiB/s, 26 objects/s recovering 2024-09-13T22:40:18.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 233376 9.24 MB/sec execute 526 sec latency 66.084 ms 2024-09-13T22:40:19.023 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:19 smithi105 ceph-mon[65554]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/1638 objects degraded (2.198%), 1 pg degraded) 2024-09-13T22:40:19.023 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:19 smithi105 ceph-mon[65554]: Cluster is now healthy 2024-09-13T22:40:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:19 smithi118 ceph-mon[51566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/1638 objects degraded (2.198%), 1 pg degraded) 2024-09-13T22:40:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:19 smithi118 ceph-mon[51566]: Cluster is now healthy 2024-09-13T22:40:19.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 233684 9.24 MB/sec execute 527 sec latency 53.691 ms 2024-09-13T22:40:20.042 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:20 smithi105 ceph-mon[65554]: pgmap v240: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 1.6 MiB/s, 20 objects/s recovering 2024-09-13T22:40:20.260 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:20 smithi118 ceph-mon[51566]: pgmap v240: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 1.6 MiB/s, 20 objects/s recovering 2024-09-13T22:40:20.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 234145 9.24 MB/sec execute 528 sec latency 38.402 ms 2024-09-13T22:40:21.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 234485 9.24 MB/sec execute 529 sec latency 44.789 ms 2024-09-13T22:40:22.079 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:22 smithi105 ceph-mon[65554]: pgmap v241: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.1 MiB/s wr, 349 op/s; 1.3 MiB/s, 16 objects/s recovering 2024-09-13T22:40:22.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:22 smithi118 ceph-mon[51566]: pgmap v241: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.1 MiB/s wr, 349 op/s; 1.3 MiB/s, 16 objects/s recovering 2024-09-13T22:40:22.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 234801 9.25 MB/sec execute 530 sec latency 59.790 ms 2024-09-13T22:40:23.126 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:23 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T22:40:23.126 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:23 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T22:40:23.126 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:23 smithi105 ceph-mon[65554]: Upgrade: osd.6 is safe to restart 2024-09-13T22:40:23.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:23 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T22:40:23.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:23 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T22:40:23.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:23 smithi118 ceph-mon[51566]: Upgrade: osd.6 is safe to restart 2024-09-13T22:40:23.609 INFO:teuthology.orchestra.run.smithi105.stdout: 5 235115 9.26 MB/sec execute 531 sec latency 35.321 ms 2024-09-13T22:40:24.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:24 smithi105 ceph-mon[65554]: pgmap v242: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 561 op/s; 1.1 MiB/s, 14 objects/s recovering 2024-09-13T22:40:24.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:24 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:24.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:24 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T22:40:24.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:24 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:40:24.065 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:24 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:24.317 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:24 smithi118 ceph-mon[51566]: pgmap v242: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 561 op/s; 1.1 MiB/s, 14 objects/s recovering 2024-09-13T22:40:24.317 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:24 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:24.317 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:24 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T22:40:24.317 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:24 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:40:24.317 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:24 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:24.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 235480 9.25 MB/sec execute 532 sec latency 62.404 ms 2024-09-13T22:40:25.104 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:25 smithi105 ceph-mon[65554]: Upgrade: Updating osd.6 2024-09-13T22:40:25.104 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:25 smithi105 ceph-mon[65554]: Deploying daemon osd.6 on smithi118 2024-09-13T22:40:25.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:25 smithi118 ceph-mon[51566]: Upgrade: Updating osd.6 2024-09-13T22:40:25.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:25 smithi118 ceph-mon[51566]: Deploying daemon osd.6 on smithi118 2024-09-13T22:40:25.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 235749 9.24 MB/sec execute 533 sec latency 60.923 ms 2024-09-13T22:40:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:26 smithi118 ceph-mon[51566]: pgmap v243: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 524 op/s; 1021 KiB/s, 12 objects/s recovering 2024-09-13T22:40:26.115 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:26 smithi105 ceph-mon[65554]: pgmap v243: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 524 op/s; 1021 KiB/s, 12 objects/s recovering 2024-09-13T22:40:26.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 235937 9.24 MB/sec execute 534 sec latency 64.805 ms 2024-09-13T22:40:27.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 236229 9.24 MB/sec execute 535 sec latency 51.224 ms 2024-09-13T22:40:27.847 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:27 smithi118 systemd[1]: Stopping Ceph osd.6 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:40:28.161 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:28 smithi105 ceph-mon[65554]: pgmap v244: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 492 op/s; 533 KiB/s, 7 objects/s recovering 2024-09-13T22:40:28.161 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:28 smithi105 ceph-mon[65554]: osd.6 marked itself down and dead 2024-09-13T22:40:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:28 smithi118 ceph-mon[51566]: pgmap v244: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 492 op/s; 533 KiB/s, 7 objects/s recovering 2024-09-13T22:40:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:28 smithi118 ceph-mon[51566]: osd.6 marked itself down and dead 2024-09-13T22:40:28.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:27 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[35838]: 2024-09-13T22:40:27.946+0000 7f8c6a227700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:40:28.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:27 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[35838]: 2024-09-13T22:40:27.946+0000 7f8c6a227700 -1 osd.6 136 *** Got signal Terminated *** 2024-09-13T22:40:28.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:27 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[35838]: 2024-09-13T22:40:27.946+0000 7f8c6a227700 -1 osd.6 136 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:40:28.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 236489 9.23 MB/sec execute 536 sec latency 582.657 ms 2024-09-13T22:40:29.052 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:28 smithi118 podman[60482]: 2024-09-13 22:40:28.639383961 +0000 UTC m=+0.889316740 container died 58849316fdafa256207a677bf2cea15bdab51b40fcb2286ed101574df010c582 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, release=754, vcs-type=git, RELEASE=HEAD, version=8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True) 2024-09-13T22:40:29.307 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:40:29.307 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: osdmap e137: 8 total, 7 up, 8 in 2024-09-13T22:40:29.307 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 2]}]: dispatch 2024-09-13T22:40:29.307 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]}]: dispatch 2024-09-13T22:40:29.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2024-09-13T22:40:29.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 6]}]: dispatch 2024-09-13T22:40:29.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 6]}]: dispatch 2024-09-13T22:40:29.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 6]}]: dispatch 2024-09-13T22:40:29.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0, 4, 6]}]: dispatch 2024-09-13T22:40:29.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 6, 1, 2]}]: dispatch 2024-09-13T22:40:29.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:29.308 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:40:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:40:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: osdmap e137: 8 total, 7 up, 8 in 2024-09-13T22:40:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 2]}]: dispatch 2024-09-13T22:40:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]}]: dispatch 2024-09-13T22:40:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2024-09-13T22:40:29.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 6]}]: dispatch 2024-09-13T22:40:29.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 6]}]: dispatch 2024-09-13T22:40:29.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 6]}]: dispatch 2024-09-13T22:40:29.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0, 4, 6]}]: dispatch 2024-09-13T22:40:29.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 6, 1, 2]}]: dispatch 2024-09-13T22:40:29.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:29.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:40:29.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 236669 9.21 MB/sec execute 537 sec latency 1050.530 ms 2024-09-13T22:40:30.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: pgmap v246: 97 pgs: 10 stale+active+clean, 87 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-13T22:40:30.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 2]}]': finished 2024-09-13T22:40:30.071 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]}]': finished 2024-09-13T22:40:30.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]': finished 2024-09-13T22:40:30.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 6]}]': finished 2024-09-13T22:40:30.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 6]}]': finished 2024-09-13T22:40:30.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 6]}]': finished 2024-09-13T22:40:30.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0, 4, 6]}]': finished 2024-09-13T22:40:30.072 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 6, 1, 2]}]': finished 2024-09-13T22:40:30.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: osdmap e138: 8 total, 7 up, 8 in 2024-09-13T22:40:30.073 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:30 smithi105 ceph-mon[65554]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T22:40:30.124 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: pgmap v246: 97 pgs: 10 stale+active+clean, 87 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-13T22:40:30.124 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 2]}]': finished 2024-09-13T22:40:30.125 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]}]': finished 2024-09-13T22:40:30.125 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]': finished 2024-09-13T22:40:30.125 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 6]}]': finished 2024-09-13T22:40:30.125 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 6]}]': finished 2024-09-13T22:40:30.125 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 6]}]': finished 2024-09-13T22:40:30.125 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 0, 4, 6]}]': finished 2024-09-13T22:40:30.125 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 6, 1, 2]}]': finished 2024-09-13T22:40:30.125 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: osdmap e138: 8 total, 7 up, 8 in 2024-09-13T22:40:30.125 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:30 smithi118 ceph-mon[51566]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T22:40:30.125 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:29 smithi118 podman[60482]: 2024-09-13 22:40:29.871038957 +0000 UTC m=+2.120971737 container cleanup 58849316fdafa256207a677bf2cea15bdab51b40fcb2286ed101574df010c582 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, ceph=True, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vendor=Red Hat, Inc.) 2024-09-13T22:40:30.125 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:29 smithi118 bash[60482]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6 2024-09-13T22:40:30.580 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:30 smithi118 podman[60497]: 2024-09-13 22:40:30.12336115 +0000 UTC m=+1.485727239 container remove 58849316fdafa256207a677bf2cea15bdab51b40fcb2286ed101574df010c582 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vcs-type=git, GIT_CLEAN=True, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, version=8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, name=centos-stream, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:40:30.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 236984 9.21 MB/sec execute 538 sec latency 37.233 ms 2024-09-13T22:40:30.847 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:30 smithi118 podman[60584]: 2024-09-13 22:40:30.481705792 +0000 UTC m=+0.018523626 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:40:30.847 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:30 smithi118 podman[60584]: 2024-09-13 22:40:30.624544617 +0000 UTC m=+0.161362440 container create 394a3070d9bd43ccad1494caa36322ae055d347d2572f90aeca823c449947144 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T22:40:31.253 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:30 smithi118 podman[60584]: 2024-09-13 22:40:30.877686162 +0000 UTC m=+0.414503986 container init 394a3070d9bd43ccad1494caa36322ae055d347d2572f90aeca823c449947144 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T22:40:31.253 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:30 smithi118 podman[60584]: 2024-09-13 22:40:30.881057895 +0000 UTC m=+0.417875728 container start 394a3070d9bd43ccad1494caa36322ae055d347d2572f90aeca823c449947144 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T22:40:31.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:31 smithi118 podman[60584]: 2024-09-13 22:40:31.253054423 +0000 UTC m=+0.789872258 container attach 394a3070d9bd43ccad1494caa36322ae055d347d2572f90aeca823c449947144 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T22:40:31.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:31 smithi118 podman[60584]: 2024-09-13 22:40:31.259934022 +0000 UTC m=+0.796751860 container died 394a3070d9bd43ccad1494caa36322ae055d347d2572f90aeca823c449947144 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-13T22:40:31.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 237347 9.22 MB/sec execute 539 sec latency 45.155 ms 2024-09-13T22:40:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:32 smithi105 ceph-mon[65554]: pgmap v249: 97 pgs: 3 peering, 3 active+undersized, 5 stale+active+clean, 11 active+undersized+degraded, 75 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 321 op/s; 75/1674 objects degraded (4.480%) 2024-09-13T22:40:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:32 smithi105 ceph-mon[65554]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T22:40:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:32 smithi105 ceph-mon[65554]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T22:40:32.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:32 smithi105 ceph-mon[65554]: Health check failed: Degraded data redundancy: 75/1674 objects degraded (4.480%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:32.297 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:32 smithi118 ceph-mon[51566]: pgmap v249: 97 pgs: 3 peering, 3 active+undersized, 5 stale+active+clean, 11 active+undersized+degraded, 75 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 321 op/s; 75/1674 objects degraded (4.480%) 2024-09-13T22:40:32.298 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:32 smithi118 ceph-mon[51566]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T22:40:32.298 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:32 smithi118 ceph-mon[51566]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T22:40:32.298 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:32 smithi118 ceph-mon[51566]: Health check failed: Degraded data redundancy: 75/1674 objects degraded (4.480%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:32.298 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 podman[60606]: 2024-09-13 22:40:32.04208709 +0000 UTC m=+0.776032295 container remove 394a3070d9bd43ccad1494caa36322ae055d347d2572f90aeca823c449947144 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T22:40:32.298 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-13T22:40:32.298 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 systemd[1]: Stopped Ceph osd.6 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:40:32.298 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.6.service: Consumed 3min 58.099s CPU time. 2024-09-13T22:40:32.560 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 systemd[1]: Starting Ceph osd.6 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:40:32.561 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 podman[60702]: 2024-09-13 22:40:32.435647436 +0000 UTC m=+0.022206122 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:40:32.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 237937 9.23 MB/sec execute 540 sec latency 50.265 ms 2024-09-13T22:40:32.823 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 podman[60702]: 2024-09-13 22:40:32.560421522 +0000 UTC m=+0.146980197 container create 6c7aa24669bbd438a4245222f63fb692639bf5446aa94aa9af13e091dbc51a01 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate, RELEASE=squid-ecab7be, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:40:33.097 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 podman[60702]: 2024-09-13 22:40:32.822436497 +0000 UTC m=+0.408995178 container init 6c7aa24669bbd438a4245222f63fb692639bf5446aa94aa9af13e091dbc51a01 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T22:40:33.097 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 podman[60702]: 2024-09-13 22:40:32.826679588 +0000 UTC m=+0.413238264 container start 6c7aa24669bbd438a4245222f63fb692639bf5446aa94aa9af13e091dbc51a01 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-13T22:40:33.097 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:32 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate[60716]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:40:33.097 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:33 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate[60716]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:40:33.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:33 smithi118 bash[60702]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:40:33.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:33 smithi118 bash[60702]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:40:33.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:33 smithi118 podman[60702]: 2024-09-13 22:40:33.155946204 +0000 UTC m=+0.742504882 container attach 6c7aa24669bbd438a4245222f63fb692639bf5446aa94aa9af13e091dbc51a01 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS) 2024-09-13T22:40:33.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 238512 9.24 MB/sec execute 541 sec latency 33.037 ms 2024-09-13T22:40:34.057 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:33 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate[60716]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T22:40:34.057 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:33 smithi118 bash[60702]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T22:40:34.057 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:33 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate[60716]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T22:40:34.057 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:33 smithi118 bash[60702]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T22:40:34.158 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:34 smithi105 ceph-mon[65554]: pgmap v251: 97 pgs: 5 active+clean+remapped, 3 peering, 8 active+undersized, 25 active+undersized+degraded, 56 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s wr, 457 op/s; 185/1581 objects degraded (11.701%); 2/1581 objects misplaced (0.127%) 2024-09-13T22:40:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:34 smithi118 ceph-mon[51566]: pgmap v251: 97 pgs: 5 active+clean+remapped, 3 peering, 8 active+undersized, 25 active+undersized+degraded, 56 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.0 MiB/s wr, 457 op/s; 185/1581 objects degraded (11.701%); 2/1581 objects misplaced (0.127%) 2024-09-13T22:40:34.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate[60716]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T22:40:34.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 bash[60702]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T22:40:34.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate[60716]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T22:40:34.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 bash[60702]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T22:40:34.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate[60716]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T22:40:34.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 bash[60702]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T22:40:34.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate[60716]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T22:40:34.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 bash[60702]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T22:40:34.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate[60716]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T22:40:34.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 bash[60702]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T22:40:34.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 podman[60702]: 2024-09-13 22:40:34.115602147 +0000 UTC m=+1.702160841 container died 6c7aa24669bbd438a4245222f63fb692639bf5446aa94aa9af13e091dbc51a01 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T22:40:34.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 238950 9.25 MB/sec execute 542 sec latency 52.042 ms 2024-09-13T22:40:34.778 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:40:35.097 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:34 smithi118 podman[60851]: 2024-09-13 22:40:34.802788802 +0000 UTC m=+0.678144442 container remove 6c7aa24669bbd438a4245222f63fb692639bf5446aa94aa9af13e091dbc51a01 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T22:40:35.097 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:35 smithi118 podman[60907]: 2024-09-13 22:40:34.921359883 +0000 UTC m=+0.017273592 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:40:35.097 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:35 smithi118 podman[60907]: 2024-09-13 22:40:35.037358385 +0000 UTC m=+0.133272080 container create e3a4ac1b8c0c38d160e6ef164903c6ced19b8ff80dac8030f83b006cafcf6cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T22:40:35.174 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:35 smithi105 ceph-mon[65554]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T22:40:35.481 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:35 smithi118 ceph-mon[51566]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T22:40:35.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 239262 9.25 MB/sec execute 543 sec latency 65.116 ms 2024-09-13T22:40:35.785 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:40:35.785 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (8m) 110s ago 17m 17.7M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:40:35.785 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (5m) 110s ago 17m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:40:35.785 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (4m) 35s ago 17m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:40:35.785 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (8m) 110s ago 17m 46.4M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:40:35.785 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (11m) 110s ago 11m 38.6M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:40:35.785 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (11m) 35s ago 11m 15.0M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (7m) 110s ago 18m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (6m) 35s ago 16m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (6m) 110s ago 19m 50.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (5m) 35s ago 16m 46.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (11m) 110s ago 11m 279M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (11m) 35s ago 11m 81.9M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (8m) 110s ago 17m 8002k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (7m) 35s ago 16m 15.2M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (4m) 110s ago 16m 327M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (3m) 110s ago 15m 270M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (2m) 110s ago 15m 189M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (2m) 110s ago 14m 12.1M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (75s) 35s ago 14m 271M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 c8d592c38cdd 2024-09-13T22:40:35.786 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (39s) 35s ago 13m 14.2M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 f03e887cfb9e 2024-09-13T22:40:35.787 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (13m) 35s ago 13m 468M 2704M 17.2.0 e1d6a67b021e 58849316fdaf 2024-09-13T22:40:35.787 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (13m) 35s ago 13m 750M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:40:35.787 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (6m) 110s ago 16m 61.0M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:40:36.203 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:36 smithi105 ceph-mon[65554]: from='client.44313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:36.203 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:36 smithi105 ceph-mon[65554]: pgmap v252: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.6 MiB/s wr, 626 op/s; 196/1653 objects degraded (11.857%); 2/1653 objects misplaced (0.121%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:36.203 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:36 smithi105 ceph-mon[65554]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:36 smithi118 ceph-mon[51566]: from='client.44313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:36 smithi118 ceph-mon[51566]: pgmap v252: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.6 MiB/s wr, 626 op/s; 196/1653 objects degraded (11.857%); 2/1653 objects misplaced (0.121%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:36 smithi118 ceph-mon[51566]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:36.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:35 smithi118 podman[60907]: 2024-09-13 22:40:35.974277379 +0000 UTC m=+1.070191082 container init e3a4ac1b8c0c38d160e6ef164903c6ced19b8ff80dac8030f83b006cafcf6cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-13T22:40:36.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:35 smithi118 podman[60907]: 2024-09-13 22:40:35.977887038 +0000 UTC m=+1.073800736 container start e3a4ac1b8c0c38d160e6ef164903c6ced19b8ff80dac8030f83b006cafcf6cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T22:40:36.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:36 smithi118 ceph-osd[60926]: -- 172.21.15.118:0/2482508893 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557d8b1701e0 con 0x557d8a35fc00 2024-09-13T22:40:36.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:36 smithi118 bash[60907]: e3a4ac1b8c0c38d160e6ef164903c6ced19b8ff80dac8030f83b006cafcf6cb6 2024-09-13T22:40:36.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:36 smithi118 systemd[1]: Started Ceph osd.6 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T22:40:36.418 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 6 2024-09-13T22:40:36.419 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:40:36.419 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:40:36.419 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:40:36.419 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:40:36.419 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:40:36.419 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T22:40:36.419 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 10 2024-09-13T22:40:36.419 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:40:36.419 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:40:36.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 239464 9.24 MB/sec execute 544 sec latency 53.069 ms 2024-09-13T22:40:36.937 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:40:36.937 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:40:36.937 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:40:36.937 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:40:36.937 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:40:36.937 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:40:36.937 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:40:36.937 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:40:36.938 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:40:36.938 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "12/23 daemons upgraded", 2024-09-13T22:40:36.938 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:40:36.938 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:40:36.938 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:40:37.219 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:37 smithi105 ceph-mon[65554]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:37.219 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:37 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:37.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:37 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:37.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:37 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:40:37.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:37 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2220343405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:37.232 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:37 smithi118 ceph-mon[51566]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:37.233 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:37 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:37.233 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:37 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:37.233 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:37 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:40:37.233 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:37 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2220343405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:37.563 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 199/1704 objects degraded (11.678%), 25 pgs degraded 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: osd.6 (root=default,host=smithi118) is down 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 199/1704 objects degraded (11.678%), 25 pgs degraded 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-13T22:40:37.564 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-13T22:40:37.565 INFO:teuthology.orchestra.run.smithi105.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-13T22:40:37.566 INFO:teuthology.orchestra.run.smithi105.stdout: pg 4.a is active+undersized+degraded, acting [2,7] 2024-09-13T22:40:37.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 239773 9.24 MB/sec execute 545 sec latency 60.770 ms 2024-09-13T22:40:38.097 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:37 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:40:37.599+0000 7fd05e6d6640 -1 Falling back to public interface 2024-09-13T22:40:38.244 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:38 smithi105 ceph-mon[65554]: pgmap v253: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 MiB/s wr, 529 op/s; 199/1704 objects degraded (11.678%); 2/1704 objects misplaced (0.117%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:38.244 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:38 smithi105 ceph-mon[65554]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:38.244 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:38 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 199/1704 objects degraded (11.678%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:38.244 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:38 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2847114840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:40:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:38 smithi118 ceph-mon[51566]: pgmap v253: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 MiB/s wr, 529 op/s; 199/1704 objects degraded (11.678%); 2/1704 objects misplaced (0.117%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:38 smithi118 ceph-mon[51566]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:40:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:38 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 199/1704 objects degraded (11.678%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:38 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2847114840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:40:38.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 240159 9.24 MB/sec execute 546 sec latency 39.932 ms 2024-09-13T22:40:39.606 INFO:teuthology.orchestra.run.smithi105.stdout: 5 240454 9.23 MB/sec execute 547 sec latency 57.391 ms 2024-09-13T22:40:40.331 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:40 smithi105 ceph-mon[65554]: pgmap v254: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 582 op/s; 199/1701 objects degraded (11.699%); 2/1701 objects misplaced (0.118%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:40 smithi118 ceph-mon[51566]: pgmap v254: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 582 op/s; 199/1701 objects degraded (11.699%); 2/1701 objects misplaced (0.118%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:40.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 240755 9.23 MB/sec execute 548 sec latency 49.051 ms 2024-09-13T22:40:41.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 241111 9.23 MB/sec execute 549 sec latency 50.545 ms 2024-09-13T22:40:42.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:41 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:42.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:41 smithi105 ceph-mon[65554]: pgmap v255: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.3 MiB/s wr, 541 op/s; 189/1623 objects degraded (11.645%); 2/1623 objects misplaced (0.123%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:42.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:41 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:42.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:41 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:42.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:41 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:42.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:41 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:42.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:41 smithi118 ceph-mon[51566]: pgmap v255: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.3 MiB/s wr, 541 op/s; 189/1623 objects degraded (11.645%); 2/1623 objects misplaced (0.123%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:42.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:41 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:42.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:41 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:42.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:41 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:42.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 241455 9.23 MB/sec execute 550 sec latency 73.531 ms 2024-09-13T22:40:43.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:42 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 181/1638 objects degraded (11.050%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:43.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:42 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 181/1638 objects degraded (11.050%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:43.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 241769 9.23 MB/sec execute 551 sec latency 48.274 ms 2024-09-13T22:40:44.141 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:43 smithi118 ceph-mon[51566]: pgmap v256: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 602 op/s; 181/1638 objects degraded (11.050%); 2/1638 objects misplaced (0.122%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:44.141 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:43 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:44.141 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:43 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:40:44.142 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:43 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:40:43.860+0000 7fd05e6d6640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-13T22:40:44.180 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:43 smithi105 ceph-mon[65554]: pgmap v256: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 602 op/s; 181/1638 objects degraded (11.050%); 2/1638 objects misplaced (0.122%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:44.180 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:43 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:44.180 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:43 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:40:44.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 242071 9.24 MB/sec execute 552 sec latency 46.928 ms 2024-09-13T22:40:45.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:40:45.127+0000 7fd05e6d6640 -1 osd.6 136 log_to_monitors true 2024-09-13T22:40:45.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 242346 9.24 MB/sec execute 553 sec latency 51.644 ms 2024-09-13T22:40:46.210 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:45 smithi105 ceph-mon[65554]: pgmap v257: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 561 op/s; 182/1668 objects degraded (10.911%); 2/1668 objects misplaced (0.120%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:46.210 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:45 smithi105 ceph-mon[65554]: from='osd.6 [v2:172.21.15.118:6816/4037347434,v1:172.21.15.118:6817/4037347434]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T22:40:46.259 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:40:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:40:45.998+0000 7fd056481640 -1 osd.6 136 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:40:46.259 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:45 smithi118 ceph-mon[51566]: pgmap v257: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 561 op/s; 182/1668 objects degraded (10.911%); 2/1668 objects misplaced (0.120%); 0 B/s, 0 objects/s recovering 2024-09-13T22:40:46.259 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:45 smithi118 ceph-mon[51566]: from='osd.6 [v2:172.21.15.118:6816/4037347434,v1:172.21.15.118:6817/4037347434]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T22:40:46.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 242664 9.24 MB/sec execute 554 sec latency 55.660 ms 2024-09-13T22:40:47.219 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='osd.6 [v2:172.21.15.118:6816/4037347434,v1:172.21.15.118:6817/4037347434]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T22:40:47.219 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: osdmap e141: 8 total, 7 up, 8 in 2024-09-13T22:40:47.219 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='osd.6 [v2:172.21.15.118:6816/4037347434,v1:172.21.15.118:6817/4037347434]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:40:47.219 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:47.219 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:47.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:40:47.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:40:47.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:47.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:47.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:40:47.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:47.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:47.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:47.220 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:46 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T22:40:47.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='osd.6 [v2:172.21.15.118:6816/4037347434,v1:172.21.15.118:6817/4037347434]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T22:40:47.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: osdmap e141: 8 total, 7 up, 8 in 2024-09-13T22:40:47.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='osd.6 [v2:172.21.15.118:6816/4037347434,v1:172.21.15.118:6817/4037347434]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:40:47.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:47.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:47.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:40:47.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:40:47.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:47.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:40:47.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:40:47.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:47.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:47.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:40:47.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:46 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T22:40:47.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 242897 9.22 MB/sec execute 555 sec latency 530.557 ms 2024-09-13T22:40:48.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:47 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T22:40:48.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:47 smithi105 ceph-mon[65554]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T22:40:48.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:47 smithi105 ceph-mon[65554]: pgmap v259: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 510 op/s; 180/1692 objects degraded (10.638%); 2/1692 objects misplaced (0.118%) 2024-09-13T22:40:48.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:47 smithi105 ceph-mon[65554]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:40:48.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:47 smithi105 ceph-mon[65554]: osd.6 [v2:172.21.15.118:6816/4037347434,v1:172.21.15.118:6817/4037347434] boot 2024-09-13T22:40:48.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:47 smithi105 ceph-mon[65554]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T22:40:48.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:40:48.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:47 smithi105 ceph-mon[65554]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T22:40:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:47 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T22:40:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:47 smithi118 ceph-mon[51566]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T22:40:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:47 smithi118 ceph-mon[51566]: pgmap v259: 97 pgs: 7 active+clean+remapped, 8 active+undersized, 25 active+undersized+degraded, 57 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 510 op/s; 180/1692 objects degraded (10.638%); 2/1692 objects misplaced (0.118%) 2024-09-13T22:40:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:47 smithi118 ceph-mon[51566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:40:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:47 smithi118 ceph-mon[51566]: osd.6 [v2:172.21.15.118:6816/4037347434,v1:172.21.15.118:6817/4037347434] boot 2024-09-13T22:40:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:47 smithi118 ceph-mon[51566]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T22:40:48.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T22:40:48.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:47 smithi118 ceph-mon[51566]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T22:40:48.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 243038 9.21 MB/sec execute 556 sec latency 1530.587 ms 2024-09-13T22:40:49.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 243290 9.20 MB/sec execute 557 sec latency 1967.351 ms 2024-09-13T22:40:50.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:49 smithi105 ceph-mon[65554]: pgmap v262: 97 pgs: 1 remapped, 4 remapped+peering, 1 unknown, 7 peering, 2 active+clean+remapped, 6 active+undersized, 19 active+undersized+degraded, 57 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.1 MiB/s wr, 349 op/s; 131/1701 objects degraded (7.701%) 2024-09-13T22:40:50.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:49 smithi105 ceph-mon[65554]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T22:40:50.080 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:49 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 131/1701 objects degraded (7.701%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:49 smithi118 ceph-mon[51566]: pgmap v262: 97 pgs: 1 remapped, 4 remapped+peering, 1 unknown, 7 peering, 2 active+clean+remapped, 6 active+undersized, 19 active+undersized+degraded, 57 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.1 MiB/s wr, 349 op/s; 131/1701 objects degraded (7.701%) 2024-09-13T22:40:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:49 smithi118 ceph-mon[51566]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T22:40:50.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:49 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 131/1701 objects degraded (7.701%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T22:40:50.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 243625 9.20 MB/sec execute 558 sec latency 589.601 ms 2024-09-13T22:40:51.102 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:51 smithi105 ceph-mon[65554]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T22:40:51.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:51 smithi118 ceph-mon[51566]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T22:40:51.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 244024 9.21 MB/sec execute 559 sec latency 1011.359 ms 2024-09-13T22:40:52.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:52 smithi105 ceph-mon[65554]: pgmap v265: 97 pgs: 2 active+undersized+remapped, 1 remapped, 4 remapped+peering, 1 unknown, 7 peering, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized, 12 active+undersized+degraded, 65 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 316 op/s; 79/1662 objects degraded (4.753%); 1.3 MiB/s, 17 objects/s recovering 2024-09-13T22:40:52.111 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:52 smithi105 ceph-mon[65554]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T22:40:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:52 smithi118 ceph-mon[51566]: pgmap v265: 97 pgs: 2 active+undersized+remapped, 1 remapped, 4 remapped+peering, 1 unknown, 7 peering, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized, 12 active+undersized+degraded, 65 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 316 op/s; 79/1662 objects degraded (4.753%); 1.3 MiB/s, 17 objects/s recovering 2024-09-13T22:40:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:52 smithi118 ceph-mon[51566]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T22:40:52.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 244351 9.20 MB/sec execute 560 sec latency 842.471 ms 2024-09-13T22:40:53.119 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:53 smithi105 ceph-mon[65554]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T22:40:53.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:53 smithi118 ceph-mon[51566]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T22:40:53.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 244689 9.20 MB/sec execute 561 sec latency 51.041 ms 2024-09-13T22:40:54.152 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:54 smithi105 ceph-mon[65554]: pgmap v268: 97 pgs: 1 active+undersized+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 620 op/s; 3/1569 objects degraded (0.191%); 784 KiB/s, 65 keys/s, 54 objects/s recovering 2024-09-13T22:40:54.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:54 smithi118 ceph-mon[51566]: pgmap v268: 97 pgs: 1 active+undersized+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 620 op/s; 3/1569 objects degraded (0.191%); 784 KiB/s, 65 keys/s, 54 objects/s recovering 2024-09-13T22:40:54.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 244918 9.20 MB/sec execute 562 sec latency 55.874 ms 2024-09-13T22:40:55.173 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:55 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 3/1569 objects degraded (0.191%), 1 pg degraded (PG_DEGRADED) 2024-09-13T22:40:55.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:55 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 3/1569 objects degraded (0.191%), 1 pg degraded (PG_DEGRADED) 2024-09-13T22:40:55.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 245177 9.21 MB/sec execute 563 sec latency 52.547 ms 2024-09-13T22:40:56.063 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:56 smithi105 ceph-mon[65554]: pgmap v269: 97 pgs: 3 peering, 94 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.2 MiB/s wr, 464 op/s; 530 KiB/s, 44 keys/s, 39 objects/s recovering 2024-09-13T22:40:56.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:56 smithi105 ceph-mon[65554]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/1569 objects degraded (0.191%), 1 pg degraded) 2024-09-13T22:40:56.064 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:56 smithi105 ceph-mon[65554]: Cluster is now healthy 2024-09-13T22:40:56.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:56 smithi118 ceph-mon[51566]: pgmap v269: 97 pgs: 3 peering, 94 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.2 MiB/s wr, 464 op/s; 530 KiB/s, 44 keys/s, 39 objects/s recovering 2024-09-13T22:40:56.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:56 smithi118 ceph-mon[51566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/1569 objects degraded (0.191%), 1 pg degraded) 2024-09-13T22:40:56.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:56 smithi118 ceph-mon[51566]: Cluster is now healthy 2024-09-13T22:40:56.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 245475 9.21 MB/sec execute 564 sec latency 50.140 ms 2024-09-13T22:40:57.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 245739 9.20 MB/sec execute 565 sec latency 66.301 ms 2024-09-13T22:40:58.246 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:58 smithi105 ceph-mon[65554]: pgmap v270: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 337 op/s; 860 KiB/s, 38 keys/s, 23 objects/s recovering 2024-09-13T22:40:58.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:58 smithi118 ceph-mon[51566]: pgmap v270: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 337 op/s; 860 KiB/s, 38 keys/s, 23 objects/s recovering 2024-09-13T22:40:58.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 245952 9.20 MB/sec execute 566 sec latency 56.170 ms 2024-09-13T22:40:59.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 246117 9.19 MB/sec execute 567 sec latency 53.894 ms 2024-09-13T22:41:00.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:00.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:41:00.055 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:40:59 smithi105 ceph-mon[65554]: pgmap v271: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 471 op/s; 742 KiB/s, 32 keys/s, 20 objects/s recovering 2024-09-13T22:41:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:41:00.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:40:59 smithi118 ceph-mon[51566]: pgmap v271: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 471 op/s; 742 KiB/s, 32 keys/s, 20 objects/s recovering 2024-09-13T22:41:00.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 246427 9.18 MB/sec execute 568 sec latency 63.411 ms 2024-09-13T22:41:01.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 246948 9.18 MB/sec execute 569 sec latency 44.039 ms 2024-09-13T22:41:02.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:01 smithi105 ceph-mon[65554]: pgmap v272: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.9 MiB/s wr, 308 op/s; 989 KiB/s, 26 keys/s, 12 objects/s recovering 2024-09-13T22:41:02.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:01 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T22:41:02.284 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:01 smithi118 ceph-mon[51566]: pgmap v272: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.9 MiB/s wr, 308 op/s; 989 KiB/s, 26 keys/s, 12 objects/s recovering 2024-09-13T22:41:02.284 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:01 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T22:41:02.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 247298 9.18 MB/sec execute 570 sec latency 45.774 ms 2024-09-13T22:41:03.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:02 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T22:41:03.068 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:02 smithi105 ceph-mon[65554]: Upgrade: osd.7 is safe to restart 2024-09-13T22:41:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:02 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T22:41:03.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:02 smithi118 ceph-mon[51566]: Upgrade: osd.7 is safe to restart 2024-09-13T22:41:03.608 INFO:teuthology.orchestra.run.smithi105.stdout: 5 247737 9.19 MB/sec execute 571 sec latency 47.983 ms 2024-09-13T22:41:04.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:03 smithi105 ceph-mon[65554]: pgmap v273: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.4 MiB/s wr, 401 op/s; 873 KiB/s, 23 keys/s, 7 objects/s recovering 2024-09-13T22:41:04.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:04.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T22:41:04.075 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:03 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:04.179 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:03 smithi118 ceph-mon[51566]: pgmap v273: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.4 MiB/s wr, 401 op/s; 873 KiB/s, 23 keys/s, 7 objects/s recovering 2024-09-13T22:41:04.179 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:04.179 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T22:41:04.179 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:03 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:04.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 248149 9.19 MB/sec execute 572 sec latency 42.622 ms 2024-09-13T22:41:05.088 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:04 smithi105 ceph-mon[65554]: Upgrade: Updating osd.7 2024-09-13T22:41:05.088 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:04 smithi105 ceph-mon[65554]: Deploying daemon osd.7 on smithi118 2024-09-13T22:41:05.088 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:04 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:05.305 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:04 smithi118 ceph-mon[51566]: Upgrade: Updating osd.7 2024-09-13T22:41:05.305 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:04 smithi118 ceph-mon[51566]: Deploying daemon osd.7 on smithi118 2024-09-13T22:41:05.305 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:04 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:05.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 248420 9.19 MB/sec execute 573 sec latency 41.720 ms 2024-09-13T22:41:06.107 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:05 smithi105 ceph-mon[65554]: pgmap v274: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 374 op/s; 682 KiB/s, 1 objects/s recovering 2024-09-13T22:41:06.287 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:05 smithi118 ceph-mon[51566]: pgmap v274: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 374 op/s; 682 KiB/s, 1 objects/s recovering 2024-09-13T22:41:06.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 248737 9.20 MB/sec execute 574 sec latency 49.928 ms 2024-09-13T22:41:06.916 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:06 smithi118 systemd[1]: Stopping Ceph osd.7 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:41:07.597 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:07 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[38904]: 2024-09-13T22:41:07.167+0000 7f5036a9c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:41:07.597 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:07 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[38904]: 2024-09-13T22:41:07.167+0000 7f5036a9c700 -1 osd.7 147 *** Got signal Terminated *** 2024-09-13T22:41:07.597 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:07 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[38904]: 2024-09-13T22:41:07.167+0000 7f5036a9c700 -1 osd.7 147 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:41:07.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 249062 9.20 MB/sec execute 575 sec latency 43.410 ms 2024-09-13T22:41:08.073 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:41:08.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:07 smithi118 ceph-mon[51566]: pgmap v275: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 447 op/s; 682 KiB/s, 0 objects/s recovering 2024-09-13T22:41:08.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:07 smithi118 ceph-mon[51566]: osd.7 marked itself down and dead 2024-09-13T22:41:08.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:07 smithi105 ceph-mon[65554]: pgmap v275: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 447 op/s; 682 KiB/s, 0 objects/s recovering 2024-09-13T22:41:08.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:07 smithi105 ceph-mon[65554]: osd.7 marked itself down and dead 2024-09-13T22:41:08.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 249197 9.19 MB/sec execute 576 sec latency 625.556 ms 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (8m) 2m ago 18m 17.7M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (5m) 2m ago 18m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (5m) 27s ago 17m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (8m) 2m ago 17m 46.4M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (12m) 2m ago 12m 38.6M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (12m) 27s ago 12m 15.2M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (8m) 2m ago 19m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (7m) 27s ago 17m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:41:08.750 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (6m) 2m ago 19m 50.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (6m) 27s ago 17m 47.0M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (11m) 2m ago 11m 279M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (11m) 27s ago 11m 82.0M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (8m) 2m ago 17m 8002k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (8m) 27s ago 17m 14.6M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (4m) 2m ago 16m 327M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (4m) 2m ago 16m 270M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (3m) 2m ago 15m 189M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (2m) 2m ago 15m 12.1M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (108s) 27s ago 14m 305M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 c8d592c38cdd 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (72s) 27s ago 14m 252M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 f03e887cfb9e 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (33s) 27s ago 14m 12.1M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 e3a4ac1b8c0c 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (13m) 27s ago 13m 779M 2704M 17.2.0 e1d6a67b021e a4fbd095bb5a 2024-09-13T22:41:08.751 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (7m) 2m ago 17m 61.0M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:41:08.976 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:08 smithi118 podman[62743]: 2024-09-13 22:41:08.665427816 +0000 UTC m=+1.823170167 container died a4fbd095bb5a9e913014deb0b5ab208187873a1261be4d2c12e0d1af67b38b50 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7, version=8, distribution-scope=public, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-type=git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, release=754, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T22:41:09.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:08 smithi105 ceph-mon[65554]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:41:09.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:08 smithi105 ceph-mon[65554]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T22:41:09.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:08 smithi105 ceph-mon[65554]: osdmap e148: 8 total, 7 up, 8 in 2024-09-13T22:41:09.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:08 smithi105 ceph-mon[65554]: from='client.44341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:09.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:08 smithi105 ceph-mon[65554]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:09.252 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:41:09.253 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:41:09.253 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:41:09.253 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:41:09.253 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:41:09.253 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:41:09.253 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:41:09.254 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:41:09.254 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 7 2024-09-13T22:41:09.254 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:41:09.254 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:41:09.254 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T22:41:09.254 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:41:09.254 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:41:09.255 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T22:41:09.255 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 11 2024-09-13T22:41:09.255 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:41:09.255 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:41:09.335 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:08 smithi118 ceph-mon[51566]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T22:41:09.336 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:08 smithi118 ceph-mon[51566]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T22:41:09.336 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:08 smithi118 ceph-mon[51566]: osdmap e148: 8 total, 7 up, 8 in 2024-09-13T22:41:09.336 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:08 smithi118 ceph-mon[51566]: from='client.44341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:09.336 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:08 smithi118 ceph-mon[51566]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:09.336 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:09 smithi118 podman[62743]: 2024-09-13 22:41:09.075231752 +0000 UTC m=+2.232974112 container cleanup a4fbd095bb5a9e913014deb0b5ab208187873a1261be4d2c12e0d1af67b38b50 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, version=8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, release=754) 2024-09-13T22:41:09.336 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:09 smithi118 bash[62743]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7 2024-09-13T22:41:09.597 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:09 smithi118 podman[62769]: 2024-09-13 22:41:09.335103459 +0000 UTC m=+0.667789588 container remove a4fbd095bb5a9e913014deb0b5ab208187873a1261be4d2c12e0d1af67b38b50 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7, architecture=x86_64, name=centos-stream, ceph=True, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, version=8, CEPH_POINT_RELEASE=-17.2.0, release=754, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.expose-services=, RELEASE=HEAD) 2024-09-13T22:41:09.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 249326 9.18 MB/sec execute 577 sec latency 1063.749 ms 2024-09-13T22:41:09.774 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:41:09.774 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:41:09.774 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:41:09.774 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:41:09.774 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:41:09.774 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:41:09.774 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:41:09.775 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:41:09.775 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:41:09.775 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "13/23 daemons upgraded", 2024-09-13T22:41:09.775 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T22:41:09.775 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:41:09.775 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:41:10.040 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:09 smithi118 ceph-mon[51566]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:10.040 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:09 smithi118 ceph-mon[51566]: pgmap v277: 97 pgs: 13 stale+active+clean, 84 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.5 MiB/s wr, 483 op/s 2024-09-13T22:41:10.040 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:09 smithi118 ceph-mon[51566]: osdmap e149: 8 total, 7 up, 8 in 2024-09-13T22:41:10.040 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:09 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/1053150610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:10.040 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:09 smithi118 podman[62857]: 2024-09-13 22:41:09.669110076 +0000 UTC m=+0.016226744 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:41:10.040 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:09 smithi118 podman[62857]: 2024-09-13 22:41:09.786053101 +0000 UTC m=+0.133169758 container create 3ba529bff73f63a1f9676b8747578076ba6e86b41993f67e85742e836c2aea93 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T22:41:10.041 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:10 smithi118 podman[62857]: 2024-09-13 22:41:10.0394423 +0000 UTC m=+0.386558958 container init 3ba529bff73f63a1f9676b8747578076ba6e86b41993f67e85742e836c2aea93 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T22:41:10.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:09 smithi105 ceph-mon[65554]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:10.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:09 smithi105 ceph-mon[65554]: pgmap v277: 97 pgs: 13 stale+active+clean, 84 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.5 MiB/s wr, 483 op/s 2024-09-13T22:41:10.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:09 smithi105 ceph-mon[65554]: osdmap e149: 8 total, 7 up, 8 in 2024-09-13T22:41:10.061 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:09 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/1053150610' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:10.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:10 smithi118 podman[62857]: 2024-09-13 22:41:10.042942442 +0000 UTC m=+0.390059101 container start 3ba529bff73f63a1f9676b8747578076ba6e86b41993f67e85742e836c2aea93 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be) 2024-09-13T22:41:10.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:10 smithi118 podman[62857]: 2024-09-13 22:41:10.131034283 +0000 UTC m=+0.478150943 container attach 3ba529bff73f63a1f9676b8747578076ba6e86b41993f67e85742e836c2aea93 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T22:41:10.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:10 smithi118 podman[62857]: 2024-09-13 22:41:10.243386781 +0000 UTC m=+0.590503447 container died 3ba529bff73f63a1f9676b8747578076ba6e86b41993f67e85742e836c2aea93 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T22:41:10.407 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid 2024-09-13T22:41:10.407 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T22:41:10.407 INFO:teuthology.orchestra.run.smithi105.stdout: osd.7 (root=default,host=smithi118) is down 2024-09-13T22:41:10.408 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-13T22:41:10.408 INFO:teuthology.orchestra.run.smithi105.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-09-13T22:41:10.608 INFO:teuthology.orchestra.run.smithi105.stdout: 5 249554 9.18 MB/sec execute 578 sec latency 51.668 ms 2024-09-13T22:41:11.085 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:11 smithi105 ceph-mon[65554]: from='client.44355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:11.085 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:11 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2664134616' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:41:11.263 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:11 smithi118 ceph-mon[51566]: from='client.44355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:11.263 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:11 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2664134616' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:41:11.264 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 podman[62880]: 2024-09-13 22:41:11.019461816 +0000 UTC m=+0.770328474 container remove 3ba529bff73f63a1f9676b8747578076ba6e86b41993f67e85742e836c2aea93 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:41:11.264 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-13T22:41:11.264 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 systemd[1]: Stopped Ceph osd.7 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:41:11.264 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 systemd[1]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.7.service: Consumed 6min 36.056s CPU time. 2024-09-13T22:41:11.521 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 systemd[1]: Starting Ceph osd.7 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:41:11.521 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 podman[62978]: 2024-09-13 22:41:11.403307351 +0000 UTC m=+0.019111430 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:41:11.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 249801 9.17 MB/sec execute 579 sec latency 34.927 ms 2024-09-13T22:41:11.847 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 podman[62978]: 2024-09-13 22:41:11.52053483 +0000 UTC m=+0.136338906 container create 2876255d279b3190dadb264204a199f807a8f7573c045170d7af6c0e4ed70156 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=) 2024-09-13T22:41:11.847 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 podman[62978]: 2024-09-13 22:41:11.757255806 +0000 UTC m=+0.373059891 container init 2876255d279b3190dadb264204a199f807a8f7573c045170d7af6c0e4ed70156 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be) 2024-09-13T22:41:11.847 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 podman[62978]: 2024-09-13 22:41:11.761510587 +0000 UTC m=+0.377314666 container start 2876255d279b3190dadb264204a199f807a8f7573c045170d7af6c0e4ed70156 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True) 2024-09-13T22:41:12.116 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:12 smithi105 ceph-mon[65554]: pgmap v279: 97 pgs: 8 stale+active+clean, 8 active+undersized+degraded, 81 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 398 op/s; 100/1671 objects degraded (5.984%) 2024-09-13T22:41:12.116 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:12 smithi105 ceph-mon[65554]: Health check failed: Degraded data redundancy: 100/1671 objects degraded (5.984%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T22:41:12.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:12 smithi118 ceph-mon[51566]: pgmap v279: 97 pgs: 8 stale+active+clean, 8 active+undersized+degraded, 81 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 398 op/s; 100/1671 objects degraded (5.984%) 2024-09-13T22:41:12.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:12 smithi118 ceph-mon[51566]: Health check failed: Degraded data redundancy: 100/1671 objects degraded (5.984%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T22:41:12.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 podman[62978]: 2024-09-13 22:41:11.865562004 +0000 UTC m=+0.481366082 container attach 2876255d279b3190dadb264204a199f807a8f7573c045170d7af6c0e4ed70156 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-13T22:41:12.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate[62993]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:41:12.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 bash[62978]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:41:12.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate[62993]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:41:12.348 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:11 smithi118 bash[62978]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T22:41:12.607 INFO:teuthology.orchestra.run.smithi105.stdout: 5 250070 9.16 MB/sec execute 580 sec latency 68.544 ms 2024-09-13T22:41:12.722 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate[62993]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T22:41:12.722 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 bash[62978]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T22:41:12.722 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate[62993]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T22:41:12.722 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 bash[62978]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T22:41:12.722 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate[62993]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T22:41:12.722 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 bash[62978]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T22:41:13.079 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate[62993]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T22:41:13.079 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 bash[62978]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T22:41:13.079 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate[62993]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T22:41:13.079 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 bash[62978]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T22:41:13.079 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate[62993]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T22:41:13.079 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 bash[62978]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T22:41:13.080 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate[62993]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T22:41:13.080 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 bash[62978]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T22:41:13.080 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:12 smithi118 podman[62978]: 2024-09-13 22:41:12.761248255 +0000 UTC m=+1.377052326 container died 2876255d279b3190dadb264204a199f807a8f7573c045170d7af6c0e4ed70156 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:41:13.608 INFO:teuthology.orchestra.run.smithi105.stdout: 5 250383 9.15 MB/sec execute 581 sec latency 45.506 ms 2024-09-13T22:41:13.630 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:13 smithi118 podman[63130]: 2024-09-13 22:41:13.378796049 +0000 UTC m=+0.606545799 container remove 2876255d279b3190dadb264204a199f807a8f7573c045170d7af6c0e4ed70156 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-13T22:41:13.630 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:13 smithi118 podman[63190]: 2024-09-13 22:41:13.508266681 +0000 UTC m=+0.018725779 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:41:13.896 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:13 smithi118 podman[63190]: 2024-09-13 22:41:13.630084018 +0000 UTC m=+0.140543107 container create baaa3d182756ae1ee4b3cb87d2213cf31e977256167d6b35e0ca4de3044d35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:41:13.897 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:13 smithi118 podman[63190]: 2024-09-13 22:41:13.866231293 +0000 UTC m=+0.376690392 container init baaa3d182756ae1ee4b3cb87d2213cf31e977256167d6b35e0ca4de3044d35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T22:41:13.897 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:13 smithi118 podman[63190]: 2024-09-13 22:41:13.869831348 +0000 UTC m=+0.380290449 container start baaa3d182756ae1ee4b3cb87d2213cf31e977256167d6b35e0ca4de3044d35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T22:41:14.207 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:14 smithi105 ceph-mon[65554]: pgmap v280: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 507 op/s; 266/1665 objects degraded (15.976%); 248 B/s, 0 objects/s recovering 2024-09-13T22:41:14.207 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:14.207 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:41:14.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:14 smithi118 ceph-mon[51566]: pgmap v280: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 507 op/s; 266/1665 objects degraded (15.976%); 248 B/s, 0 objects/s recovering 2024-09-13T22:41:14.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:14.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:41:14.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:13 smithi118 ceph-osd[63209]: -- 172.21.15.118:0/336118551 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563cde7ac1e0 con 0x563cdd999c00 2024-09-13T22:41:14.348 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:13 smithi118 bash[63190]: baaa3d182756ae1ee4b3cb87d2213cf31e977256167d6b35e0ca4de3044d35a5 2024-09-13T22:41:14.348 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:13 smithi118 systemd[1]: Started Ceph osd.7 for 782bdfc8-721e-11ef-bceb-c7b262605968. 2024-09-13T22:41:14.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 250678 9.15 MB/sec execute 582 sec latency 44.011 ms 2024-09-13T22:41:15.222 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:15 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:15.222 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:15 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:15.222 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:15 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:41:15.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:15 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:15.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:15 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:15.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:15 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:41:15.348 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:15 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:41:15.252+0000 7f7e94967640 -1 Falling back to public interface 2024-09-13T22:41:15.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 251147 9.16 MB/sec execute 583 sec latency 32.638 ms 2024-09-13T22:41:16.226 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:16 smithi105 ceph-mon[65554]: pgmap v281: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s wr, 386 op/s; 255/1635 objects degraded (15.596%); 248 B/s, 0 objects/s recovering 2024-09-13T22:41:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:16 smithi118 ceph-mon[51566]: pgmap v281: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s wr, 386 op/s; 255/1635 objects degraded (15.596%); 248 B/s, 0 objects/s recovering 2024-09-13T22:41:16.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 251653 9.16 MB/sec execute 584 sec latency 33.504 ms 2024-09-13T22:41:17.257 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:17 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 256/1647 objects degraded (15.543%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T22:41:17.263 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:17 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 256/1647 objects degraded (15.543%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T22:41:17.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 252003 9.17 MB/sec execute 585 sec latency 42.457 ms 2024-09-13T22:41:18.273 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:18 smithi105 ceph-mon[65554]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 436 op/s; 256/1647 objects degraded (15.543%); 222 B/s, 0 objects/s recovering 2024-09-13T22:41:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:18 smithi118 ceph-mon[51566]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 436 op/s; 256/1647 objects degraded (15.543%); 222 B/s, 0 objects/s recovering 2024-09-13T22:41:18.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 252638 9.19 MB/sec execute 586 sec latency 29.469 ms 2024-09-13T22:41:19.538 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:19 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:19.538 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:19 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:19.538 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:19 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:19.539 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:19 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:19.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 253173 9.20 MB/sec execute 587 sec latency 29.250 ms 2024-09-13T22:41:19.684 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:19 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:19.684 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:19 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:19.684 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:19 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:19.684 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:19 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:20.546 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:20 smithi105 ceph-mon[65554]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 427 op/s; 252/1689 objects degraded (14.920%); 198 B/s, 0 objects/s recovering 2024-09-13T22:41:20.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 253558 9.20 MB/sec execute 588 sec latency 30.736 ms 2024-09-13T22:41:20.724 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:20 smithi118 ceph-mon[51566]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 427 op/s; 252/1689 objects degraded (14.920%); 198 B/s, 0 objects/s recovering 2024-09-13T22:41:21.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 253899 9.19 MB/sec execute 589 sec latency 72.399 ms 2024-09-13T22:41:22.584 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:22 smithi105 ceph-mon[65554]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.6 MiB/s wr, 463 op/s; 256/1698 objects degraded (15.077%); 166 B/s, 0 objects/s recovering 2024-09-13T22:41:22.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 254125 9.18 MB/sec execute 590 sec latency 59.790 ms 2024-09-13T22:41:22.838 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:22 smithi118 ceph-mon[51566]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.6 MiB/s wr, 463 op/s; 256/1698 objects degraded (15.077%); 166 B/s, 0 objects/s recovering 2024-09-13T22:41:23.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:22 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:41:22.836+0000 7f7e94967640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-13T22:41:23.597 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:23 smithi105 ceph-mon[65554]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.1 MiB/s wr, 609 op/s; 263/1743 objects degraded (15.089%); 165 B/s, 0 objects/s recovering 2024-09-13T22:41:23.597 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:23 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 263/1743 objects degraded (15.089%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T22:41:23.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 254412 9.18 MB/sec execute 591 sec latency 83.260 ms 2024-09-13T22:41:23.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:23 smithi118 ceph-mon[51566]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.1 MiB/s wr, 609 op/s; 263/1743 objects degraded (15.089%); 165 B/s, 0 objects/s recovering 2024-09-13T22:41:23.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:23 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 263/1743 objects degraded (15.089%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T22:41:24.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 254709 9.18 MB/sec execute 592 sec latency 60.854 ms 2024-09-13T22:41:24.847 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:24 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:41:24.583+0000 7f7e94967640 -1 osd.7 147 log_to_monitors true 2024-09-13T22:41:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:24 smithi118 ceph-mon[51566]: from='osd.7 [v2:172.21.15.118:6824/480252103,v1:172.21.15.118:6825/480252103]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T22:41:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:24 smithi118 ceph-mon[51566]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T22:41:24.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:24 smithi105 ceph-mon[65554]: from='osd.7 [v2:172.21.15.118:6824/480252103,v1:172.21.15.118:6825/480252103]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T22:41:24.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:24 smithi105 ceph-mon[65554]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T22:41:25.230 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:41:24 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:41:24.951+0000 7f7e8870a640 -1 osd.7 147 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T22:41:25.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 255039 9.18 MB/sec execute 593 sec latency 72.710 ms 2024-09-13T22:41:25.674 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:25 smithi105 ceph-mon[65554]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T22:41:25.674 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:25 smithi105 ceph-mon[65554]: osdmap e150: 8 total, 7 up, 8 in 2024-09-13T22:41:25.674 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:25 smithi105 ceph-mon[65554]: from='osd.7 [v2:172.21.15.118:6824/480252103,v1:172.21.15.118:6825/480252103]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:41:25.674 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:25 smithi105 ceph-mon[65554]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:41:25.675 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:25 smithi105 ceph-mon[65554]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.9 MiB/s wr, 614 op/s; 259/1701 objects degraded (15.226%) 2024-09-13T22:41:25.675 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:25 smithi105 ceph-mon[65554]: from='osd.7 ' entity='osd.7' 2024-09-13T22:41:25.675 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:25 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:25 smithi118 ceph-mon[51566]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T22:41:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:25 smithi118 ceph-mon[51566]: osdmap e150: 8 total, 7 up, 8 in 2024-09-13T22:41:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:25 smithi118 ceph-mon[51566]: from='osd.7 [v2:172.21.15.118:6824/480252103,v1:172.21.15.118:6825/480252103]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:41:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:25 smithi118 ceph-mon[51566]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-13T22:41:26.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:25 smithi118 ceph-mon[51566]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.9 MiB/s wr, 614 op/s; 259/1701 objects degraded (15.226%) 2024-09-13T22:41:26.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:25 smithi118 ceph-mon[51566]: from='osd.7 ' entity='osd.7' 2024-09-13T22:41:26.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:25 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:26.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 255197 9.17 MB/sec execute 594 sec latency 658.819 ms 2024-09-13T22:41:26.975 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:26.975 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:26.975 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:41:26.975 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:26.975 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:26.975 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:41:26.975 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all osd 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T22:41:26.976 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T22:41:26.977 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: osd.7 [v2:172.21.15.118:6824/480252103,v1:172.21.15.118:6825/480252103] boot 2024-09-13T22:41:26.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T22:41:26.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:41:26.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi105.ddoxtb"]}]: dispatch 2024-09-13T22:41:26.978 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:26 smithi105 ceph-mon[65554]: Upgrade: It appears safe to stop mds.foofs.smithi105.ddoxtb 2024-09-13T22:41:27.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:27.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:27.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all osd 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-13T22:41:27.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T22:41:27.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T22:41:27.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-13T22:41:27.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T22:41:27.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T22:41:27.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T22:41:27.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: osd.7 [v2:172.21.15.118:6824/480252103,v1:172.21.15.118:6825/480252103] boot 2024-09-13T22:41:27.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T22:41:27.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T22:41:27.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi105.ddoxtb"]}]: dispatch 2024-09-13T22:41:27.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:26 smithi118 ceph-mon[51566]: Upgrade: It appears safe to stop mds.foofs.smithi105.ddoxtb 2024-09-13T22:41:27.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 255303 9.16 MB/sec execute 595 sec latency 1631.680 ms 2024-09-13T22:41:28.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:27 smithi105 ceph-mon[65554]: pgmap v289: 97 pgs: 1 unknown, 16 peering, 1 active+undersized, 24 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 622 op/s; 190/1647 objects degraded (11.536%) 2024-09-13T22:41:28.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:27 smithi105 ceph-mon[65554]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T22:41:28.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:28.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi105.ddoxtb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:41:28.105 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:27 smithi118 ceph-mon[51566]: pgmap v289: 97 pgs: 1 unknown, 16 peering, 1 active+undersized, 24 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 622 op/s; 190/1647 objects degraded (11.536%) 2024-09-13T22:41:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:27 smithi118 ceph-mon[51566]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T22:41:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi105.ddoxtb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:41:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:28.618 INFO:teuthology.orchestra.run.smithi105.stdout: 5 255489 9.15 MB/sec execute 596 sec latency 2116.533 ms 2024-09-13T22:41:29.130 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:28 smithi105 ceph-mon[65554]: Upgrade: Updating mds.foofs.smithi105.ddoxtb 2024-09-13T22:41:29.130 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:28 smithi105 ceph-mon[65554]: Deploying daemon mds.foofs.smithi105.ddoxtb on smithi105 2024-09-13T22:41:29.130 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:28 smithi105 ceph-mon[65554]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T22:41:29.130 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:29.130 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:28 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:41:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:28 smithi118 ceph-mon[51566]: Upgrade: Updating mds.foofs.smithi105.ddoxtb 2024-09-13T22:41:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:28 smithi118 ceph-mon[51566]: Deploying daemon mds.foofs.smithi105.ddoxtb on smithi105 2024-09-13T22:41:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:28 smithi118 ceph-mon[51566]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T22:41:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:28 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:41:29.618 INFO:teuthology.orchestra.run.smithi105.stdout: 5 255699 9.15 MB/sec execute 597 sec latency 636.198 ms 2024-09-13T22:41:30.177 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:29 smithi105 ceph-mon[65554]: pgmap v292: 97 pgs: 1 active+recovering+degraded, 3 remapped+peering, 1 unknown, 16 peering, 1 active+recovery_wait+degraded, 14 active+undersized+degraded, 61 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 MiB/s wr, 462 op/s; 140/1593 objects degraded (8.788%); 18 KiB/s, 3 objects/s recovering 2024-09-13T22:41:30.177 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:29 smithi105 ceph-mon[65554]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T22:41:30.177 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:29 smithi105 ceph-mon[65554]: Health check update: Degraded data redundancy: 190/1647 objects degraded (11.536%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T22:41:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:29 smithi118 ceph-mon[51566]: pgmap v292: 97 pgs: 1 active+recovering+degraded, 3 remapped+peering, 1 unknown, 16 peering, 1 active+recovery_wait+degraded, 14 active+undersized+degraded, 61 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 MiB/s wr, 462 op/s; 140/1593 objects degraded (8.788%); 18 KiB/s, 3 objects/s recovering 2024-09-13T22:41:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:29 smithi118 ceph-mon[51566]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T22:41:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:29 smithi118 ceph-mon[51566]: Health check update: Degraded data redundancy: 190/1647 objects degraded (11.536%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T22:41:30.617 INFO:teuthology.orchestra.run.smithi105.stdout: 5 256012 9.16 MB/sec execute 598 sec latency 1014.809 ms 2024-09-13T22:41:30.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105[65550]: 2024-09-13T22:41:30.586+0000 7f24f68e4640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T22:41:30.993 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:41:30.617+0000 7fb4b8496640 -1 bdev(0x563a2a1f6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-09-13T22:41:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-mon[51566]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T22:41:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-mon[51566]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T22:41:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-mon[51566]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T22:41:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-mon[51566]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T22:41:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-mon[51566]: Standby daemon mds.foofs.smithi118.izbofj assigned to filesystem foofs as rank 0 2024-09-13T22:41:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-mon[51566]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T22:41:31.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-mon[51566]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T22:41:31.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-mon[51566]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T22:41:31.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:30 smithi118 ceph-mon[51566]: fsmap foofs:1/1 {0=foofs.smithi118.izbofj=up:replay} 2024-09-13T22:41:31.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-mon[65554]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T22:41:31.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-mon[65554]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T22:41:31.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-mon[65554]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T22:41:31.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-mon[65554]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T22:41:31.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-mon[65554]: Standby daemon mds.foofs.smithi118.izbofj assigned to filesystem foofs as rank 0 2024-09-13T22:41:31.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-mon[65554]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T22:41:31.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-mon[65554]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T22:41:31.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-mon[65554]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T22:41:31.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:30 smithi105 ceph-mon[65554]: fsmap foofs:1/1 {0=foofs.smithi118.izbofj=up:replay} 2024-09-13T22:41:31.618 INFO:teuthology.orchestra.run.smithi105.stdout: 5 256012 9.14 MB/sec execute 599 sec latency 1034.911 ms 2024-09-13T22:41:32.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:31 smithi118 ceph-mon[51566]: pgmap v296: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 1 unknown, 21 peering, 1 active+recovery_wait+degraded, 72 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s wr, 183 op/s; 26/1590 objects degraded (1.635%); 1.7 MiB/s, 48 objects/s recovering 2024-09-13T22:41:32.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:31 smithi105 ceph-mon[65554]: pgmap v296: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 1 unknown, 21 peering, 1 active+recovery_wait+degraded, 72 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s wr, 183 op/s; 26/1590 objects degraded (1.635%); 1.7 MiB/s, 48 objects/s recovering 2024-09-13T22:41:32.620 INFO:teuthology.orchestra.run.smithi105.stdout: 5 cleanup 600 sec 2024-09-13T22:41:33.349 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:33 smithi118 ceph-mon[51566]: reconnect by client.24333 172.21.15.118:0/3226915716 after 0 2024-09-13T22:41:33.350 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:33 smithi118 ceph-mon[51566]: reconnect by client.14548 172.21.15.105:0/570452982 after 0 2024-09-13T22:41:33.350 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:33 smithi118 ceph-mon[51566]: mds.? [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] up:reconnect 2024-09-13T22:41:33.350 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:33 smithi118 ceph-mon[51566]: fsmap foofs:1/1 {0=foofs.smithi118.izbofj=up:reconnect} 2024-09-13T22:41:33.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:33 smithi105 ceph-mon[65554]: reconnect by client.24333 172.21.15.118:0/3226915716 after 0 2024-09-13T22:41:33.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:33 smithi105 ceph-mon[65554]: reconnect by client.14548 172.21.15.105:0/570452982 after 0 2024-09-13T22:41:33.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:33 smithi105 ceph-mon[65554]: mds.? [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] up:reconnect 2024-09-13T22:41:33.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:33 smithi105 ceph-mon[65554]: fsmap foofs:1/1 {0=foofs.smithi118.izbofj=up:reconnect} 2024-09-13T22:41:33.618 INFO:teuthology.orchestra.run.smithi105.stdout: 5 cleanup 601 sec 2024-09-13T22:41:34.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:34 smithi105 ceph-mon[65554]: pgmap v297: 97 pgs: 5 peering, 92 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 2.7 MiB/s wr, 211 op/s; 2.9 MiB/s, 52 objects/s recovering 2024-09-13T22:41:34.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:34 smithi105 ceph-mon[65554]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1590 objects degraded (1.635%), 2 pgs degraded) 2024-09-13T22:41:34.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:34 smithi105 ceph-mon[65554]: mds.? [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] up:rejoin 2024-09-13T22:41:34.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:34 smithi105 ceph-mon[65554]: fsmap foofs:1/1 {0=foofs.smithi118.izbofj=up:rejoin} 2024-09-13T22:41:34.266 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:34 smithi105 ceph-mon[65554]: daemon mds.foofs.smithi118.izbofj is now active in filesystem foofs as rank 0 2024-09-13T22:41:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:34 smithi118 ceph-mon[51566]: pgmap v297: 97 pgs: 5 peering, 92 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 2.7 MiB/s wr, 211 op/s; 2.9 MiB/s, 52 objects/s recovering 2024-09-13T22:41:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:34 smithi118 ceph-mon[51566]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1590 objects degraded (1.635%), 2 pgs degraded) 2024-09-13T22:41:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:34 smithi118 ceph-mon[51566]: mds.? [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] up:rejoin 2024-09-13T22:41:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:34 smithi118 ceph-mon[51566]: fsmap foofs:1/1 {0=foofs.smithi118.izbofj=up:rejoin} 2024-09-13T22:41:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:34 smithi118 ceph-mon[51566]: daemon mds.foofs.smithi118.izbofj is now active in filesystem foofs as rank 0 2024-09-13T22:41:34.618 INFO:teuthology.orchestra.run.smithi105.stdout: 5 cleanup 602 sec 2024-09-13T22:41:35.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:35 smithi105 ceph-mon[65554]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T22:41:35.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:35 smithi105 ceph-mon[65554]: mds.? [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] up:active 2024-09-13T22:41:35.060 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:35 smithi105 ceph-mon[65554]: fsmap foofs:1 {0=foofs.smithi118.izbofj=up:active} 2024-09-13T22:41:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:35 smithi118 ceph-mon[51566]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T22:41:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:35 smithi118 ceph-mon[51566]: mds.? [v2:172.21.15.118:6832/3411221692,v1:172.21.15.118:6833/3411221692] up:active 2024-09-13T22:41:35.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:35 smithi118 ceph-mon[51566]: fsmap foofs:1 {0=foofs.smithi118.izbofj=up:active} 2024-09-13T22:41:35.273 INFO:teuthology.orchestra.run.smithi105.stdout: 0 cleanup 603 sec 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: Operation Count AvgLat MaxLat 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: ---------------------------------------- 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: NTCreateX 175732 4.856 3461.954 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: Close 129004 0.126 15.066 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: Rename 7448 8.898 2259.392 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: Unlink 35548 7.475 2152.291 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: Qpathinfo 159341 1.197 2092.897 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: Qfileinfo 27768 0.006 0.736 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: Qfsinfo 29210 0.631 60.748 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: Sfileinfo 14315 13.637 2270.549 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: Find 61507 0.882 23.216 2024-09-13T22:41:35.274 INFO:teuthology.orchestra.run.smithi105.stdout: WriteX 86789 15.450 3434.782 2024-09-13T22:41:35.275 INFO:teuthology.orchestra.run.smithi105.stdout: ReadX 275619 0.010 3.398 2024-09-13T22:41:35.275 INFO:teuthology.orchestra.run.smithi105.stdout: LockX 574 0.345 2.289 2024-09-13T22:41:35.275 INFO:teuthology.orchestra.run.smithi105.stdout: UnlockX 574 0.297 2.739 2024-09-13T22:41:35.275 INFO:teuthology.orchestra.run.smithi105.stdout: Flush 12308 0.002 0.023 2024-09-13T22:41:35.275 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:41:35.275 INFO:teuthology.orchestra.run.smithi105.stdout:Throughput 9.13977 MB/sec 5 clients 5 procs max_latency=2659.194 ms 2024-09-13T22:41:35.276 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-13T22:41:35.322 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-13T22:41:35.393 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-13T22:41:35.418 INFO:teuthology.orchestra.run.smithi105.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-13T22:41:35.418 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:41:35.418 INFO:teuthology.orchestra.run.smithi105.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-13T22:41:35.419 INFO:teuthology.orchestra.run.smithi105.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-13T22:41:35.519 INFO:teuthology.orchestra.run.smithi105.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-13T22:41:35.519 INFO:teuthology.orchestra.run.smithi105.stdout:releasing clients 2024-09-13T22:41:36.163 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:36 smithi105 ceph-mon[65554]: pgmap v298: 97 pgs: 97 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 2.8 MiB/s wr, 234 op/s; 2.4 MiB/s, 43 objects/s recovering 2024-09-13T22:41:36.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:36 smithi118 ceph-mon[51566]: pgmap v298: 97 pgs: 97 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 2.8 MiB/s wr, 234 op/s; 2.4 MiB/s, 43 objects/s recovering 2024-09-13T22:41:37.519 INFO:teuthology.orchestra.run.smithi105.stdout: 5 261 17.95 MB/sec execute 1 sec latency 113.777 ms 2024-09-13T22:41:38.249 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:38 smithi105 ceph-mon[65554]: pgmap v299: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 3.2 MiB/s wr, 220 op/s; 1.8 MiB/s, 33 objects/s recovering 2024-09-13T22:41:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:38 smithi118 ceph-mon[51566]: pgmap v299: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 3.2 MiB/s wr, 220 op/s; 1.8 MiB/s, 33 objects/s recovering 2024-09-13T22:41:38.519 INFO:teuthology.orchestra.run.smithi105.stdout: 5 357 16.26 MB/sec execute 2 sec latency 129.999 ms 2024-09-13T22:41:39.519 INFO:teuthology.orchestra.run.smithi105.stdout: 5 443 15.52 MB/sec execute 3 sec latency 155.999 ms 2024-09-13T22:41:40.164 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:40 smithi105 ceph-mon[65554]: pgmap v300: 97 pgs: 97 active+clean; 301 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.5 MiB/s wr, 168 op/s; 902 KiB/s, 9 objects/s recovering 2024-09-13T22:41:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:40 smithi118 ceph-mon[51566]: pgmap v300: 97 pgs: 97 active+clean; 301 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.5 MiB/s wr, 168 op/s; 902 KiB/s, 9 objects/s recovering 2024-09-13T22:41:40.519 INFO:teuthology.orchestra.run.smithi105.stdout: 5 568 16.96 MB/sec execute 4 sec latency 123.678 ms 2024-09-13T22:41:40.895 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:41:41.519 INFO:teuthology.orchestra.run.smithi105.stdout: 5 cleanup 5 sec 2024-09-13T22:41:41.699 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:41:41.699 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (9m) 2m ago 18m 17.7M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:41:41.699 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (6m) 2m ago 18m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:41:41.699 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (6m) 23s ago 18m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:41:41.700 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (9m) 2m ago 18m 46.4M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:41:41.700 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (12m) 2m ago 12m 38.6M - 17.2.0 e1d6a67b021e d6d662402f93 2024-09-13T22:41:41.700 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (12m) 23s ago 12m 15.3M - 17.2.0 e1d6a67b021e f7a8c3ba56c7 2024-09-13T22:41:41.700 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (8m) 2m ago 19m 505M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:41:41.700 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (7m) 23s ago 18m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:41:41.700 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (7m) 2m ago 20m 50.6M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:41:41.700 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (6m) 23s ago 17m 45.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:41:41.700 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (12m) 2m ago 12m 279M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:41:41.700 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (12m) 23s ago 12m 82.0M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:41:41.701 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (9m) 2m ago 18m 8002k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:41:41.701 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (8m) 23s ago 17m 14.7M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:41:41.701 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (5m) 2m ago 17m 327M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:41:41.701 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (4m) 2m ago 16m 270M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:41:41.701 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (3m) 2m ago 16m 189M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:41:41.701 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (3m) 2m ago 15m 12.1M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:41:41.701 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (2m) 23s ago 15m 336M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 c8d592c38cdd 2024-09-13T22:41:41.702 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (105s) 23s ago 15m 291M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 f03e887cfb9e 2024-09-13T22:41:41.702 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (66s) 23s ago 14m 251M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 e3a4ac1b8c0c 2024-09-13T22:41:41.702 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (28s) 23s ago 14m 12.1M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 baaa3d182756 2024-09-13T22:41:41.702 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (7m) 2m ago 17m 61.0M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:41:42.195 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:41:42.195 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:41:42.195 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:41:42.196 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:41:42.197 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T22:41:42.197 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 12 2024-09-13T22:41:42.197 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:41:42.197 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:41:42.232 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:42 smithi105 ceph-mon[65554]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:42.232 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:42 smithi105 ceph-mon[65554]: pgmap v301: 97 pgs: 97 active+clean; 293 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 5.6 MiB/s wr, 180 op/s; 782 KiB/s, 8 objects/s recovering 2024-09-13T22:41:42.232 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:42 smithi105 ceph-mon[65554]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: 0 cleanup 6 sec 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: Operation Count AvgLat MaxLat 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: ---------------------------------------- 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: NTCreateX 417 27.734 155.996 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: Close 399 13.178 93.081 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: Rename 9 35.111 66.432 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: Unlink 20 8.156 21.245 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: Qpathinfo 58 0.556 2.125 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: Qfileinfo 323 17.957 52.311 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: Qfsinfo 65 0.468 2.640 2024-09-13T22:41:42.274 INFO:teuthology.orchestra.run.smithi105.stdout: Sfileinfo 36 18.085 32.204 2024-09-13T22:41:42.275 INFO:teuthology.orchestra.run.smithi105.stdout: Find 50 0.872 2.835 2024-09-13T22:41:42.275 INFO:teuthology.orchestra.run.smithi105.stdout: WriteX 1548 0.027 0.067 2024-09-13T22:41:42.275 INFO:teuthology.orchestra.run.smithi105.stdout: ReadX 202 0.016 0.253 2024-09-13T22:41:42.275 INFO:teuthology.orchestra.run.smithi105.stdout: Flush 67 18.239 35.508 2024-09-13T22:41:42.275 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:41:42.275 INFO:teuthology.orchestra.run.smithi105.stdout:Throughput 16.9589 MB/sec 5 clients 5 procs max_latency=155.999 ms 2024-09-13T22:41:42.276 DEBUG:teuthology.parallel:result is None 2024-09-13T22:41:42.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:42 smithi118 ceph-mon[51566]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:42.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:42 smithi118 ceph-mon[51566]: pgmap v301: 97 pgs: 97 active+clean; 293 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 5.6 MiB/s wr, 180 op/s; 782 KiB/s, 8 objects/s recovering 2024-09-13T22:41:42.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:42 smithi118 ceph-mon[51566]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:42.569 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:41:42.569 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:41:42.569 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:41:42.569 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:41:42.569 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:41:42.569 INFO:teuthology.orchestra.run.smithi105.stdout: "osd", 2024-09-13T22:41:42.570 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:41:42.570 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:41:42.570 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr" 2024-09-13T22:41:42.570 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:41:42.570 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "14/23 daemons upgraded", 2024-09-13T22:41:42.570 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading mds daemons", 2024-09-13T22:41:42.570 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:41:42.570 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:41:43.008 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_WARN insufficient standby MDS daemons available 2024-09-13T22:41:43.008 INFO:teuthology.orchestra.run.smithi105.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-13T22:41:43.008 INFO:teuthology.orchestra.run.smithi105.stdout: have 0; want 1 more 2024-09-13T22:41:43.100 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:43 smithi105 ceph-mon[65554]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:43.100 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:43 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2420936600' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:43.100 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:43 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2304486171' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:41:43.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:43 smithi118 ceph-mon[51566]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:43.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:43 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2420936600' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:43.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:43 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2304486171' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:41:44.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:44.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: pgmap v302: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 9.4 MiB/s wr, 295 op/s; 673 KiB/s, 7 objects/s recovering 2024-09-13T22:41:44.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: Cluster is now healthy 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347] up:boot 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: fsmap foofs:1 {0=foofs.smithi118.izbofj=up:active} 1 up:standby 2024-09-13T22:41:44.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:41:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:41:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: pgmap v302: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 9.4 MiB/s wr, 295 op/s; 673 KiB/s, 7 objects/s recovering 2024-09-13T22:41:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:41:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:41:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:44.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T22:41:44.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: Cluster is now healthy 2024-09-13T22:41:44.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347] up:boot 2024-09-13T22:41:44.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: fsmap foofs:1 {0=foofs.smithi118.izbofj=up:active} 1 up:standby 2024-09-13T22:41:44.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:41:46.623 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:46 smithi105 ceph-mon[65554]: pgmap v303: 97 pgs: 97 active+clean; 304 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 9.3 MiB/s wr, 262 op/s 2024-09-13T22:41:46.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:46 smithi118 ceph-mon[51566]: pgmap v303: 97 pgs: 97 active+clean; 304 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 9.3 MiB/s wr, 262 op/s 2024-09-13T22:41:48.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:48 smithi105 ceph-mon[65554]: pgmap v304: 97 pgs: 97 active+clean; 260 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 9.1 MiB/s wr, 245 op/s 2024-09-13T22:41:48.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:48 smithi118 ceph-mon[51566]: pgmap v304: 97 pgs: 97 active+clean; 260 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 9.1 MiB/s wr, 245 op/s 2024-09-13T22:41:49.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:49.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:49.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:49 smithi118 ceph-mon[51566]: pgmap v305: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 8.4 MiB/s wr, 226 op/s 2024-09-13T22:41:49.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:49.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:49 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:49.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:49.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:49.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:49 smithi105 ceph-mon[65554]: pgmap v305: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 8.4 MiB/s wr, 226 op/s 2024-09-13T22:41:49.868 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:49.869 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:49 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:52.221 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:51 smithi105 ceph-mon[65554]: pgmap v306: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.5 MiB/s wr, 207 op/s 2024-09-13T22:41:52.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:51 smithi118 ceph-mon[51566]: pgmap v306: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.5 MiB/s wr, 207 op/s 2024-09-13T22:41:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: Detected new or changed devices on smithi105 2024-09-13T22:41:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:53.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi118.izbofj"]}]: dispatch 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: Upgrade: It appears safe to stop mds.foofs.smithi118.izbofj 2024-09-13T22:41:53.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:53 smithi118 ceph-mon[51566]: pgmap v307: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.3 MiB/s wr, 178 op/s 2024-09-13T22:41:53.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: Detected new or changed devices on smithi105 2024-09-13T22:41:53.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:53.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:53.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:53.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:41:53.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:53.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:53.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:41:53.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:53.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:53.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:53.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:41:53.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi118.izbofj"]}]: dispatch 2024-09-13T22:41:53.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: Upgrade: It appears safe to stop mds.foofs.smithi118.izbofj 2024-09-13T22:41:53.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:53 smithi105 ceph-mon[65554]: pgmap v307: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.3 MiB/s wr, 178 op/s 2024-09-13T22:41:55.585 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:55 smithi118 ceph-mon[51566]: Upgrade: Updating mds.foofs.smithi118.izbofj 2024-09-13T22:41:55.585 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:55 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:55.585 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:55 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.izbofj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:41:55.585 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:55 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:55.585 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:55 smithi118 ceph-mon[51566]: Deploying daemon mds.foofs.smithi118.izbofj on smithi118 2024-09-13T22:41:55.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:55 smithi105 ceph-mon[65554]: Upgrade: Updating mds.foofs.smithi118.izbofj 2024-09-13T22:41:55.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:55 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:55.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:55 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.izbofj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T22:41:55.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:55 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:41:55.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:55 smithi105 ceph-mon[65554]: Deploying daemon mds.foofs.smithi118.izbofj on smithi118 2024-09-13T22:41:56.582 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:56 smithi118 ceph-mon[51566]: pgmap v308: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 783 KiB/s wr, 38 op/s 2024-09-13T22:41:56.583 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:56 smithi105 ceph-mon[65554]: pgmap v308: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 783 KiB/s wr, 38 op/s 2024-09-13T22:41:56.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:41:56 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:41:56.608+0000 7fb4bdca1640 -1 bdev(0x563a2a1f6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-13T22:41:56.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:41:56 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:41:56.609+0000 7fb4b8496640 -1 bdev(0x563a2a1f6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 6 2024-09-13T22:41:56.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:41:56 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:41:56.611+0000 7fb4b8c97640 -1 bdev(0x563a2a1f6c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 12 2024-09-13T22:41:56.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:56 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105[65550]: 2024-09-13T22:41:56.581+0000 7f24f68e4640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T22:41:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:57 smithi118 ceph-mon[51566]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T22:41:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:57 smithi118 ceph-mon[51566]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T22:41:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:57 smithi118 ceph-mon[51566]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T22:41:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:57 smithi118 ceph-mon[51566]: Standby daemon mds.foofs.smithi105.ddoxtb assigned to filesystem foofs as rank 0 2024-09-13T22:41:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:57 smithi118 ceph-mon[51566]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T22:41:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:57 smithi118 ceph-mon[51566]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T22:41:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:57 smithi118 ceph-mon[51566]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T22:41:57.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:57 smithi118 ceph-mon[51566]: fsmap foofs:1/1 {0=foofs.smithi105.ddoxtb=up:replay} 2024-09-13T22:41:57.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:57 smithi105 ceph-mon[65554]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T22:41:57.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:57 smithi105 ceph-mon[65554]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T22:41:57.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:57 smithi105 ceph-mon[65554]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T22:41:57.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:57 smithi105 ceph-mon[65554]: Standby daemon mds.foofs.smithi105.ddoxtb assigned to filesystem foofs as rank 0 2024-09-13T22:41:57.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:57 smithi105 ceph-mon[65554]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T22:41:57.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:57 smithi105 ceph-mon[65554]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T22:41:57.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:57 smithi105 ceph-mon[65554]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T22:41:57.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:57 smithi105 ceph-mon[65554]: fsmap foofs:1/1 {0=foofs.smithi105.ddoxtb=up:replay} 2024-09-13T22:41:58.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:58 smithi118 ceph-mon[51566]: pgmap v310: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 60 KiB/s wr, 13 op/s 2024-09-13T22:41:58.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:58 smithi105 ceph-mon[65554]: pgmap v310: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 60 KiB/s wr, 13 op/s 2024-09-13T22:41:59.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:59 smithi118 ceph-mon[51566]: reconnect by client.14548 172.21.15.105:0/570452982 after 0 2024-09-13T22:41:59.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:59 smithi118 ceph-mon[51566]: reconnect by client.24333 172.21.15.118:0/3226915716 after 0 2024-09-13T22:41:59.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:59 smithi118 ceph-mon[51566]: mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347] up:reconnect 2024-09-13T22:41:59.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:59 smithi118 ceph-mon[51566]: fsmap foofs:1/1 {0=foofs.smithi105.ddoxtb=up:reconnect} 2024-09-13T22:41:59.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:59.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:41:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:41:59.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:59 smithi105 ceph-mon[65554]: reconnect by client.14548 172.21.15.105:0/570452982 after 0 2024-09-13T22:41:59.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:59 smithi105 ceph-mon[65554]: reconnect by client.24333 172.21.15.118:0/3226915716 after 0 2024-09-13T22:41:59.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:59 smithi105 ceph-mon[65554]: mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347] up:reconnect 2024-09-13T22:41:59.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:59 smithi105 ceph-mon[65554]: fsmap foofs:1/1 {0=foofs.smithi105.ddoxtb=up:reconnect} 2024-09-13T22:41:59.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:41:59.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:41:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:42:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:00 smithi118 ceph-mon[51566]: pgmap v311: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 4.4 KiB/s wr, 4 op/s 2024-09-13T22:42:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:00 smithi118 ceph-mon[51566]: mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347] up:rejoin 2024-09-13T22:42:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:00 smithi118 ceph-mon[51566]: fsmap foofs:1/1 {0=foofs.smithi105.ddoxtb=up:rejoin} 2024-09-13T22:42:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:00 smithi118 ceph-mon[51566]: daemon mds.foofs.smithi105.ddoxtb is now active in filesystem foofs as rank 0 2024-09-13T22:42:00.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:42:00.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:00 smithi105 ceph-mon[65554]: pgmap v311: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 4.4 KiB/s wr, 4 op/s 2024-09-13T22:42:00.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:00 smithi105 ceph-mon[65554]: mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347] up:rejoin 2024-09-13T22:42:00.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:00 smithi105 ceph-mon[65554]: fsmap foofs:1/1 {0=foofs.smithi105.ddoxtb=up:rejoin} 2024-09-13T22:42:00.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:00 smithi105 ceph-mon[65554]: daemon mds.foofs.smithi105.ddoxtb is now active in filesystem foofs as rank 0 2024-09-13T22:42:00.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:42:01.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:01 smithi118 ceph-mon[51566]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T22:42:01.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:01 smithi118 ceph-mon[51566]: mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347] up:active 2024-09-13T22:42:01.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:01 smithi118 ceph-mon[51566]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 2024-09-13T22:42:01.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:01 smithi105 ceph-mon[65554]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T22:42:01.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:01 smithi105 ceph-mon[65554]: mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347] up:active 2024-09-13T22:42:01.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:01 smithi105 ceph-mon[65554]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 2024-09-13T22:42:02.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:02 smithi118 ceph-mon[51566]: pgmap v312: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.4 KiB/s wr, 5 op/s 2024-09-13T22:42:02.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:02 smithi105 ceph-mon[65554]: pgmap v312: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.4 KiB/s wr, 5 op/s 2024-09-13T22:42:04.332 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:04 smithi118 ceph-mon[51566]: pgmap v313: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 409 B/s wr, 10 op/s 2024-09-13T22:42:04.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:04 smithi105 ceph-mon[65554]: pgmap v313: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 409 B/s wr, 10 op/s 2024-09-13T22:42:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:06 smithi118 ceph-mon[51566]: pgmap v314: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 409 B/s wr, 11 op/s 2024-09-13T22:42:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:06 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:06 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:06.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:06 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:06.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:06 smithi105 ceph-mon[65554]: pgmap v314: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 409 B/s wr, 11 op/s 2024-09-13T22:42:06.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:06 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:06.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:06 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:06.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:06 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:07.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:07 smithi118 ceph-mon[51566]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T22:42:07.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:07 smithi118 ceph-mon[51566]: Cluster is now healthy 2024-09-13T22:42:07.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:07 smithi118 ceph-mon[51566]: mds.? [v2:172.21.15.118:6832/1365305812,v1:172.21.15.118:6833/1365305812] up:boot 2024-09-13T22:42:07.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:07 smithi118 ceph-mon[51566]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:42:07.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:07 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:42:07.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:07 smithi105 ceph-mon[65554]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T22:42:07.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:07 smithi105 ceph-mon[65554]: Cluster is now healthy 2024-09-13T22:42:07.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:07 smithi105 ceph-mon[65554]: mds.? [v2:172.21.15.118:6832/1365305812,v1:172.21.15.118:6833/1365305812] up:boot 2024-09-13T22:42:07.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:07 smithi105 ceph-mon[65554]: fsmap foofs:1 {0=foofs.smithi105.ddoxtb=up:active} 1 up:standby 2024-09-13T22:42:07.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:07 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:42:08.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:08 smithi118 ceph-mon[51566]: pgmap v315: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 396 B/s wr, 10 op/s 2024-09-13T22:42:08.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:08 smithi105 ceph-mon[65554]: pgmap v315: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 396 B/s wr, 10 op/s 2024-09-13T22:42:09.744 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:09.744 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:09.744 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:09.744 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:09 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:09.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:09.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:09.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:09.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:09 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:10.770 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:10 smithi118 ceph-mon[51566]: pgmap v316: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.4 KiB/s wr, 12 op/s 2024-09-13T22:42:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:10 smithi105 ceph-mon[65554]: pgmap v316: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.4 KiB/s wr, 12 op/s 2024-09-13T22:42:11.816 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:11 smithi118 ceph-mon[51566]: pgmap v317: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 7.2 KiB/s wr, 10 op/s 2024-09-13T22:42:11.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:11 smithi105 ceph-mon[65554]: pgmap v317: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 7.2 KiB/s wr, 10 op/s 2024-09-13T22:42:13.383 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: Detected new or changed devices on smithi118 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all mds 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi105.ddoxtb"}]': finished 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi118.izbofj"}]': finished 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all rgw 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:42:13.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: pgmap v318: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-09-13T22:42:13.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:13.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:42:13.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:13.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:13.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all iscsi 2024-09-13T22:42:13.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:13.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:13 smithi105 ceph-mon[65554]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:14.075 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:42:14.075 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (10m) 25s ago 19m 28.2M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:42:14.075 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (6m) 25s ago 19m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:42:14.075 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (6m) 5s ago 18m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (9m) 25s ago 18m 53.7M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (31s) 25s ago 13m 10.9M - 19.1.1-324-gecab7be5 0652ff2b5d78 a36b4be60260 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (8s) 5s ago 13m 11.4M - 19.1.1-324-gecab7be5 0652ff2b5d78 c42e4784af3e 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (9m) 25s ago 20m 509M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (8m) 5s ago 18m 442M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (8m) 25s ago 20m 60.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (7m) 5s ago 18m 48.2M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (12m) 25s ago 12m 280M - 4.0 e1d6a67b021e 71a3646b0130 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (12m) 5s ago 12m 82.3M - 4.0 e1d6a67b021e 556431d5ac6b 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (9m) 25s ago 18m 8497k - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (9m) 5s ago 18m 15.3M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:42:14.076 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (5m) 25s ago 17m 394M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:42:14.077 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (5m) 25s ago 17m 329M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:42:14.077 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (4m) 25s ago 16m 367M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:42:14.077 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (3m) 25s ago 16m 400M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:42:14.077 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (2m) 5s ago 15m 367M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 c8d592c38cdd 2024-09-13T22:42:14.077 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (2m) 5s ago 15m 332M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 f03e887cfb9e 2024-09-13T22:42:14.077 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (99s) 5s ago 15m 280M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 e3a4ac1b8c0c 2024-09-13T22:42:14.077 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (60s) 5s ago 14m 302M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 baaa3d182756 2024-09-13T22:42:14.077 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (8m) 25s ago 18m 62.2M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:42:14.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: Detected new or changed devices on smithi118 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all mds 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi105.ddoxtb"}]: dispatch 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi105.ddoxtb"}]': finished 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi118.izbofj"}]: dispatch 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi118.izbofj"}]': finished 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all rgw 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: pgmap v318: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-09-13T22:42:14.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:14.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T22:42:14.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:14.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:14.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all iscsi 2024-09-13T22:42:14.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:14.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:13 smithi118 ceph-mon[51566]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:14.487 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:42:14.487 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:42:14.487 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:42:14.487 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:42:14.487 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 14 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:42:14.488 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:42:14.847 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:42:14.847 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:42:14.847 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:42:14.847 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:42:14.847 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:42:14.847 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:42:14.847 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:42:14.847 INFO:teuthology.orchestra.run.smithi105.stdout: "mds", 2024-09-13T22:42:14.848 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr", 2024-09-13T22:42:14.848 INFO:teuthology.orchestra.run.smithi105.stdout: "osd" 2024-09-13T22:42:14.848 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:42:14.848 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "16/23 daemons upgraded", 2024-09-13T22:42:14.848 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading nfs daemons", 2024-09-13T22:42:14.848 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:42:14.848 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:42:15.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:15.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:15.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:42:15.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:15.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/4293602978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:15.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:15.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T22:42:15.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T22:42:15.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T22:42:15.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:15.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T22:42:15.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T22:42:15.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:42:15.141 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:15.279 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:42:15.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:15.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:15.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:42:15.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:15.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/4293602978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:15.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:15.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T22:42:15.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T22:42:15.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T22:42:15.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:15.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T22:42:15.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T22:42:15.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi105.xxfcdx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:42:15.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:16.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: Upgrade: Updating nfs.foo.0.0.smithi105.xxfcdx (1/2) 2024-09-13T22:42:16.044 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: Creating key for client.nfs.foo.0.0.smithi105.xxfcdx 2024-09-13T22:42:16.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T22:42:16.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: Rados config object exists: conf-nfs.foo 2024-09-13T22:42:16.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: Creating key for client.nfs.foo.0.0.smithi105.xxfcdx-rgw 2024-09-13T22:42:16.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: Bind address in nfs.foo.0.0.smithi105.xxfcdx's ganesha conf is defaulting to empty 2024-09-13T22:42:16.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: Deploying daemon nfs.foo.0.0.smithi105.xxfcdx on smithi105 2024-09-13T22:42:16.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:16.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: pgmap v319: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-13T22:42:16.045 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:15 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/3743596302' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:42:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: Upgrade: Updating nfs.foo.0.0.smithi105.xxfcdx (1/2) 2024-09-13T22:42:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: Creating key for client.nfs.foo.0.0.smithi105.xxfcdx 2024-09-13T22:42:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T22:42:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: Rados config object exists: conf-nfs.foo 2024-09-13T22:42:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: Creating key for client.nfs.foo.0.0.smithi105.xxfcdx-rgw 2024-09-13T22:42:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: Bind address in nfs.foo.0.0.smithi105.xxfcdx's ganesha conf is defaulting to empty 2024-09-13T22:42:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: Deploying daemon nfs.foo.0.0.smithi105.xxfcdx on smithi105 2024-09-13T22:42:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: pgmap v319: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-13T22:42:16.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:15 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/3743596302' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:42:18.304 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:17 smithi105 ceph-mon[65554]: pgmap v320: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-13T22:42:18.320 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:17 smithi118 ceph-mon[51566]: pgmap v320: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-13T22:42:19.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:19 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:19.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:19 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:19.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:19 smithi105 ceph-mon[65554]: Evicting (and blocklisting) client session 14548 (172.21.15.105:0/570452982) 2024-09-13T22:42:19.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:19 smithi105 ceph-mon[65554]: from='mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347]' entity='mds.foofs.smithi105.ddoxtb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.105:0/570452982"}]: dispatch 2024-09-13T22:42:19.792 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:19 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:19.793 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:19 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:19.793 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:19 smithi118 ceph-mon[51566]: Evicting (and blocklisting) client session 14548 (172.21.15.105:0/570452982) 2024-09-13T22:42:19.793 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:19 smithi118 ceph-mon[51566]: from='mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347]' entity='mds.foofs.smithi105.ddoxtb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.105:0/570452982"}]: dispatch 2024-09-13T22:42:20.661 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: pgmap v321: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-09-13T22:42:20.661 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347]' entity='mds.foofs.smithi105.ddoxtb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.105:0/570452982"}]': finished 2024-09-13T22:42:20.661 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T22:42:20.662 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:20.662 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T22:42:20.662 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T22:42:20.662 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T22:42:20.662 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:20.662 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T22:42:20.662 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T22:42:20.662 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:42:20.662 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:20 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:20.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: pgmap v321: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-09-13T22:42:20.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347]' entity='mds.foofs.smithi105.ddoxtb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.105:0/570452982"}]': finished 2024-09-13T22:42:20.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T22:42:20.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:20.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T22:42:20.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T22:42:20.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T22:42:20.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:20.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T22:42:20.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T22:42:20.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.chlqvt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T22:42:20.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:20 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:21.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:21 smithi118 ceph-mon[51566]: Upgrade: Updating nfs.foo.1.0.smithi118.chlqvt (2/2) 2024-09-13T22:42:21.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:21 smithi118 ceph-mon[51566]: Creating key for client.nfs.foo.1.0.smithi118.chlqvt 2024-09-13T22:42:21.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:21 smithi118 ceph-mon[51566]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T22:42:21.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:21 smithi118 ceph-mon[51566]: Rados config object exists: conf-nfs.foo 2024-09-13T22:42:21.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:21 smithi118 ceph-mon[51566]: Creating key for client.nfs.foo.1.0.smithi118.chlqvt-rgw 2024-09-13T22:42:21.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:21 smithi118 ceph-mon[51566]: Bind address in nfs.foo.1.0.smithi118.chlqvt's ganesha conf is defaulting to empty 2024-09-13T22:42:21.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:21 smithi118 ceph-mon[51566]: Deploying daemon nfs.foo.1.0.smithi118.chlqvt on smithi118 2024-09-13T22:42:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:21 smithi105 ceph-mon[65554]: Upgrade: Updating nfs.foo.1.0.smithi118.chlqvt (2/2) 2024-09-13T22:42:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:21 smithi105 ceph-mon[65554]: Creating key for client.nfs.foo.1.0.smithi118.chlqvt 2024-09-13T22:42:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:21 smithi105 ceph-mon[65554]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T22:42:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:21 smithi105 ceph-mon[65554]: Rados config object exists: conf-nfs.foo 2024-09-13T22:42:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:21 smithi105 ceph-mon[65554]: Creating key for client.nfs.foo.1.0.smithi118.chlqvt-rgw 2024-09-13T22:42:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:21 smithi105 ceph-mon[65554]: Bind address in nfs.foo.1.0.smithi118.chlqvt's ganesha conf is defaulting to empty 2024-09-13T22:42:21.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:21 smithi105 ceph-mon[65554]: Deploying daemon nfs.foo.1.0.smithi118.chlqvt on smithi118 2024-09-13T22:42:22.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:22 smithi118 ceph-mon[51566]: pgmap v323: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 307 B/s wr, 0 op/s 2024-09-13T22:42:22.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:22 smithi105 ceph-mon[65554]: pgmap v323: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 307 B/s wr, 0 op/s 2024-09-13T22:42:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:24 smithi118 ceph-mon[51566]: pgmap v324: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-13T22:42:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:24 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:24 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:24 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:24.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:24 smithi105 ceph-mon[65554]: pgmap v324: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-13T22:42:24.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:24 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:24.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:24 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:24.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:24 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:26.816 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:26 smithi105 ceph-mon[65554]: pgmap v325: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-13T22:42:26.843 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:26 smithi118 ceph-mon[51566]: pgmap v325: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-13T22:42:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:27 smithi105 ceph-mon[65554]: Evicting (and blocklisting) client session 24333 (172.21.15.118:0/3226915716) 2024-09-13T22:42:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:27 smithi105 ceph-mon[65554]: from='mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347]' entity='mds.foofs.smithi105.ddoxtb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/3226915716"}]: dispatch 2024-09-13T22:42:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:27 smithi105 ceph-mon[65554]: pgmap v326: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-13T22:42:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:27.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:27 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:27.943 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:27.943 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:27.944 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:27 smithi118 ceph-mon[51566]: Evicting (and blocklisting) client session 24333 (172.21.15.118:0/3226915716) 2024-09-13T22:42:27.944 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:27 smithi118 ceph-mon[51566]: from='mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347]' entity='mds.foofs.smithi105.ddoxtb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/3226915716"}]: dispatch 2024-09-13T22:42:27.944 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:27 smithi118 ceph-mon[51566]: pgmap v326: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-13T22:42:27.944 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:27.944 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:27 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:29.010 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:28 smithi118 ceph-mon[51566]: from='mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347]' entity='mds.foofs.smithi105.ddoxtb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/3226915716"}]': finished 2024-09-13T22:42:29.010 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:28 smithi118 ceph-mon[51566]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T22:42:29.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:28 smithi105 ceph-mon[65554]: from='mds.? [v2:172.21.15.105:6834/1959761347,v1:172.21.15.105:6835/1959761347]' entity='mds.foofs.smithi105.ddoxtb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/3226915716"}]': finished 2024-09-13T22:42:29.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:28 smithi105 ceph-mon[65554]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T22:42:30.039 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:30.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:30.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:42:30.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:29 smithi105 ceph-mon[65554]: pgmap v328: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-09-13T22:42:30.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:30.040 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:30.048 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:30.048 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:30.048 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:42:30.049 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:29 smithi118 ceph-mon[51566]: pgmap v328: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-09-13T22:42:30.049 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:30.049 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:31.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:31 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:31.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:31 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:31.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:31 smithi105 ceph-mon[65554]: pgmap v329: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-13T22:42:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:31 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:31 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:32.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:31 smithi118 ceph-mon[51566]: pgmap v329: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: pgmap v330: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.391 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all nfs 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi105.xxfcdx"}]: dispatch 2024-09-13T22:42:34.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi105.xxfcdx"}]': finished 2024-09-13T22:42:34.393 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi118.chlqvt"}]: dispatch 2024-09-13T22:42:34.393 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi118.chlqvt"}]': finished 2024-09-13T22:42:34.393 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.393 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:42:34.393 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.393 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.393 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.393 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:34 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: pgmap v330: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all nfs 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi105.xxfcdx"}]: dispatch 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi105.xxfcdx"}]': finished 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi118.chlqvt"}]: dispatch 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi118.chlqvt"}]': finished 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: Upgrade: Setting container_image for all nvmeof 2024-09-13T22:42:34.599 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:34.600 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.600 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:34.600 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:34 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:36.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:36 smithi118 ceph-mon[51566]: Upgrade: Updating grafana.smithi105 2024-09-13T22:42:36.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:36 smithi118 ceph-mon[51566]: Deploying daemon grafana.smithi105 on smithi105 2024-09-13T22:42:36.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:36 smithi118 ceph-mon[51566]: pgmap v331: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-13T22:42:36.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:36 smithi105 ceph-mon[65554]: Upgrade: Updating grafana.smithi105 2024-09-13T22:42:36.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:36 smithi105 ceph-mon[65554]: Deploying daemon grafana.smithi105 on smithi105 2024-09-13T22:42:36.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:36 smithi105 ceph-mon[65554]: pgmap v331: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-13T22:42:38.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:38 smithi118 ceph-mon[51566]: pgmap v332: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 409 B/s wr, 0 op/s 2024-09-13T22:42:38.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:38 smithi105 ceph-mon[65554]: pgmap v332: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 409 B/s wr, 0 op/s 2024-09-13T22:42:40.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:40 smithi118 ceph-mon[51566]: pgmap v333: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 91 B/s rd, 273 B/s wr, 0 op/s 2024-09-13T22:42:40.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:40 smithi105 ceph-mon[65554]: pgmap v333: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 91 B/s rd, 273 B/s wr, 0 op/s 2024-09-13T22:42:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:42 smithi118 ceph-mon[51566]: pgmap v334: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T22:42:42.632 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:42 smithi105 ceph-mon[65554]: pgmap v334: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T22:42:44.307 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:44 smithi105 ceph-mon[65554]: pgmap v335: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T22:42:44.307 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:42:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:44 smithi118 ceph-mon[51566]: pgmap v335: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T22:42:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:42:45.691 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:42:46.406 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:42:46.406 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (10m) 17s ago 19m 27.4M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:42:46.407 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (7m) 17s ago 19m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:42:46.407 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (7m) 19s ago 19m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:42:46.407 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (10m) 17s ago 19m 55.6M - 8.3.5 dad864ee21e9 14e99a952346 2024-09-13T22:42:46.407 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (63s) 17s ago 13m 36.7M - 19.1.1-324-gecab7be5 0652ff2b5d78 a36b4be60260 2024-09-13T22:42:46.407 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (41s) 19s ago 13m 12.8M - 19.1.1-324-gecab7be5 0652ff2b5d78 c42e4784af3e 2024-09-13T22:42:46.407 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (9m) 17s ago 20m 511M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:42:46.407 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (8m) 19s ago 19m 443M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:42:46.407 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (8m) 17s ago 21m 62.5M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:42:46.408 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (7m) 19s ago 19m 48.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:42:46.408 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (28s) 17s ago 13m 60.0M - 5.5 0652ff2b5d78 b6b7bc689a93 2024-09-13T22:42:46.408 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (23s) 19s ago 13m 17.5M - 5.5 0652ff2b5d78 a247799bcbba 2024-09-13T22:42:46.408 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (10m) 17s ago 19m 15.0M - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:42:46.408 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (9m) 19s ago 18m 15.5M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:42:46.408 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (6m) 17s ago 18m 396M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:42:46.408 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (5m) 17s ago 17m 328M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:42:46.408 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (5m) 17s ago 17m 369M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:42:46.409 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (4m) 17s ago 16m 401M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:42:46.409 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (3m) 19s ago 16m 368M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 c8d592c38cdd 2024-09-13T22:42:46.409 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (2m) 19s ago 16m 333M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 f03e887cfb9e 2024-09-13T22:42:46.409 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (2m) 19s ago 15m 281M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 e3a4ac1b8c0c 2024-09-13T22:42:46.409 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (92s) 19s ago 15m 302M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 baaa3d182756 2024-09-13T22:42:46.409 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (8m) 17s ago 18m 62.3M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:42:46.444 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:46 smithi105 ceph-mon[65554]: pgmap v336: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:46.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:46 smithi118 ceph-mon[51566]: pgmap v336: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:46.902 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:42:46.902 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:42:46.902 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:42:46.902 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:42:46.902 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:42:46.902 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:42:46.902 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:42:46.902 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:42:46.903 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T22:42:46.903 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:42:46.903 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:42:46.903 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:42:46.903 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:42:46.903 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:42:46.903 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 14 2024-09-13T22:42:46.903 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:42:46.903 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": true, 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [ 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout: "mon", 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout: "crash", 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout: "mds", 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout: "nfs", 2024-09-13T22:42:47.314 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr", 2024-09-13T22:42:47.315 INFO:teuthology.orchestra.run.smithi105.stdout: "osd" 2024-09-13T22:42:47.315 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:42:47.315 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": "18/23 daemons upgraded", 2024-09-13T22:42:47.315 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "Currently upgrading grafana daemons", 2024-09-13T22:42:47.315 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:42:47.315 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:42:47.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:47 smithi105 ceph-mon[65554]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:47.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:47 smithi105 ceph-mon[65554]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:47.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:47 smithi105 ceph-mon[65554]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:47.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:47.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:47.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:47 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:47.469 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:47 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/988788505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:47.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:47 smithi118 ceph-mon[51566]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:47.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:47 smithi118 ceph-mon[51566]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:47.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:47 smithi118 ceph-mon[51566]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:47.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:47.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:47.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:47 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:47.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:47 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/988788505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:47.751 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:42:48.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:48 smithi118 ceph-mon[51566]: pgmap v337: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:48.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:48 smithi118 ceph-mon[51566]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:48.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:48 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2962835674' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:42:48.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:48 smithi105 ceph-mon[65554]: pgmap v337: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:48.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:48 smithi105 ceph-mon[65554]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:42:48.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:48 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2962835674' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:42:50.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:50 smithi105 ceph-mon[65554]: pgmap v338: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:50.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:50 smithi118 ceph-mon[51566]: pgmap v338: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:52.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:52 smithi105 ceph-mon[65554]: pgmap v339: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:52.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:52 smithi118 ceph-mon[51566]: pgmap v339: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:53.581 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:53.581 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:53.581 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:53.581 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:53 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:53.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:53.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:53.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:53.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:53 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:54.620 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:54 smithi105 ceph-mon[65554]: pgmap v340: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:42:54.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:54 smithi118 ceph-mon[51566]: pgmap v340: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:42:56.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:56 smithi105 ceph-mon[65554]: pgmap v341: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:56.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:56 smithi118 ceph-mon[51566]: pgmap v341: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:58.567 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:58 smithi105 ceph-mon[65554]: pgmap v342: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:58.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:58 smithi118 ceph-mon[51566]: pgmap v342: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:59.891 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.892 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: Upgrade: Finalizing container_image settings 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:42:59.893 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:42:59.894 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: Upgrade: Complete! 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:59.895 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: pgmap v343: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:42:59.896 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:42:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.097 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.098 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: Upgrade: Finalizing container_image settings 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.099 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T22:43:00.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T22:43:00.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T22:43:00.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:43:00.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T22:43:00.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T22:43:00.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T22:43:00.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T22:43:00.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T22:43:00.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: Upgrade: Complete! 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T22:43:00.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: pgmap v343: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:00.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:42:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:43:02.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:01 smithi118 ceph-mon[51566]: pgmap v344: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:02.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:01 smithi105 ceph-mon[65554]: pgmap v344: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:04.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:03 smithi118 ceph-mon[51566]: pgmap v345: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:04.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:03 smithi105 ceph-mon[65554]: pgmap v345: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:06.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:05 smithi118 ceph-mon[51566]: pgmap v346: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:06.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:05 smithi105 ceph-mon[65554]: pgmap v346: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:08.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:07 smithi105 ceph-mon[65554]: pgmap v347: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:08.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:07 smithi118 ceph-mon[51566]: pgmap v347: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:10.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:10 smithi118 ceph-mon[51566]: pgmap v348: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:10.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:10 smithi105 ceph-mon[65554]: pgmap v348: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:12.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:12 smithi118 ceph-mon[51566]: pgmap v349: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:12.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:12 smithi105 ceph-mon[65554]: pgmap v349: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:14.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:14 smithi118 ceph-mon[51566]: pgmap v350: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:14.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:43:14.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:14 smithi105 ceph-mon[65554]: pgmap v350: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:14.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:43:16.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:16 smithi118 ceph-mon[51566]: pgmap v351: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:16.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:16 smithi105 ceph-mon[65554]: pgmap v351: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:18.115 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:18 smithi105 ceph-mon[65554]: pgmap v352: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:18.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:18 smithi118 ceph-mon[51566]: pgmap v352: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:18.744 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T22:43:19.289 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:19 smithi105 ceph-mon[65554]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:43:19.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:19 smithi118 ceph-mon[51566]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:43:20.290 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:20 smithi105 ceph-mon[65554]: pgmap v353: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:20.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:20 smithi118 ceph-mon[51566]: pgmap v353: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:20.732 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:43:20.732 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (11m) 28s ago 20m 27.4M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:43:20.732 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (7m) 28s ago 20m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:43:20.732 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (7m) 54s ago 19m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:43:20.732 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (34s) 28s ago 20m 42.0M - 9.4.12 2bacad6d85d8 76bb146938c4 2024-09-13T22:43:20.733 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (98s) 28s ago 14m 36.7M - 19.1.1-324-gecab7be5 0652ff2b5d78 a36b4be60260 2024-09-13T22:43:20.733 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (75s) 54s ago 14m 12.8M - 19.1.1-324-gecab7be5 0652ff2b5d78 c42e4784af3e 2024-09-13T22:43:20.733 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (10m) 28s ago 21m 519M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:43:20.733 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (9m) 54s ago 19m 443M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:43:20.733 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (9m) 28s ago 21m 63.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:43:20.733 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (8m) 54s ago 19m 48.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:43:20.733 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (62s) 28s ago 14m 60.3M - 5.5 0652ff2b5d78 b6b7bc689a93 2024-09-13T22:43:20.734 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (57s) 54s ago 13m 17.5M - 5.5 0652ff2b5d78 a247799bcbba 2024-09-13T22:43:20.734 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (10m) 28s ago 20m 15.4M - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:43:20.734 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (10m) 54s ago 19m 15.5M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:43:20.734 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (7m) 28s ago 18m 396M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:43:20.734 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (6m) 28s ago 18m 328M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:43:20.734 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (5m) 28s ago 17m 369M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:43:20.735 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (4m) 28s ago 17m 402M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:43:20.735 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (4m) 54s ago 17m 368M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 c8d592c38cdd 2024-09-13T22:43:20.735 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (3m) 54s ago 16m 333M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 f03e887cfb9e 2024-09-13T22:43:20.735 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (2m) 54s ago 16m 281M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 e3a4ac1b8c0c 2024-09-13T22:43:20.735 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (2m) 54s ago 15m 302M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 baaa3d182756 2024-09-13T22:43:20.735 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (9m) 28s ago 19m 62.6M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:43:21.450 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T22:43:22.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:22 smithi118 ceph-mon[51566]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:43:22.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:22 smithi118 ceph-mon[51566]: pgmap v354: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:22.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:22 smithi105 ceph-mon[65554]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:43:22.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:22 smithi105 ceph-mon[65554]: pgmap v354: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:23.565 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:43:23.565 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:43:23.565 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:43:23.565 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:43:23.565 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:43:23.565 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:43:23.565 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:43:23.565 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:43:23.565 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T22:43:23.566 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:43:23.566 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:43:23.566 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:43:23.566 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:43:23.566 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:43:23.566 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 14 2024-09-13T22:43:23.566 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:43:23.566 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:43:24.224 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:24 smithi105 ceph-mon[65554]: pgmap v355: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:24.224 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:24 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/3539966696' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:24.248 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-13T22:43:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:24 smithi118 ceph-mon[51566]: pgmap v355: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:24.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:24 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/3539966696' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:43:26.067 INFO:teuthology.orchestra.run.smithi105.stdout:wait for servicemap items w/ changing names to refresh 2024-09-13T22:43:26.140 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:26 smithi105 ceph-mon[65554]: pgmap v356: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:26.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:26 smithi118 ceph-mon[51566]: pgmap v356: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:26.750 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'sleep 60' 2024-09-13T22:43:28.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:28 smithi118 ceph-mon[51566]: pgmap v357: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:28.372 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:28 smithi105 ceph-mon[65554]: pgmap v357: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:29.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:43:29.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:43:30.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:30 smithi118 ceph-mon[51566]: pgmap v358: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:30.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:30 smithi105 ceph-mon[65554]: pgmap v358: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:32.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:32 smithi118 ceph-mon[51566]: pgmap v359: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:32.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:32 smithi105 ceph-mon[65554]: pgmap v359: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:34.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:34 smithi118 ceph-mon[51566]: pgmap v360: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:34.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:34 smithi105 ceph-mon[65554]: pgmap v360: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:36.348 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:36 smithi118 ceph-mon[51566]: pgmap v361: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:36.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:36 smithi105 ceph-mon[65554]: pgmap v361: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:38.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:38 smithi118 ceph-mon[51566]: pgmap v362: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:38.392 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:38 smithi105 ceph-mon[65554]: pgmap v362: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:40.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:40 smithi118 ceph-mon[51566]: pgmap v363: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:40.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:40 smithi105 ceph-mon[65554]: pgmap v363: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:42.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:42 smithi105 ceph-mon[65554]: pgmap v364: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:42.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:42 smithi118 ceph-mon[51566]: pgmap v364: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:44.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:44 smithi105 ceph-mon[65554]: pgmap v365: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:44.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:43:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:44 smithi118 ceph-mon[51566]: pgmap v365: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:44.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:43:46.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:46 smithi105 ceph-mon[65554]: pgmap v366: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:46.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:46 smithi118 ceph-mon[51566]: pgmap v366: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:48.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:48 smithi105 ceph-mon[65554]: pgmap v367: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:48.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:48 smithi118 ceph-mon[51566]: pgmap v367: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:50.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:50 smithi105 ceph-mon[65554]: pgmap v368: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:50.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:50 smithi118 ceph-mon[51566]: pgmap v368: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:52.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:52 smithi105 ceph-mon[65554]: pgmap v369: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:52.399 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:52 smithi118 ceph-mon[51566]: pgmap v369: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:54.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:54 smithi105 ceph-mon[65554]: pgmap v370: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:54.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:54 smithi118 ceph-mon[51566]: pgmap v370: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:43:56.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:56 smithi105 ceph-mon[65554]: pgmap v371: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:56.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:56 smithi118 ceph-mon[51566]: pgmap v371: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:58.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:58 smithi105 ceph-mon[65554]: pgmap v372: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:58.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:58 smithi118 ceph-mon[51566]: pgmap v372: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:43:59.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:43:59.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:43:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:43:59.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:43:59.597 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:43:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T22:44:00.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:00 smithi105 ceph-mon[65554]: pgmap v373: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:00.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:44:00.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:44:00.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:00 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:00 smithi118 ceph-mon[51566]: pgmap v373: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:00.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T22:44:00.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T22:44:00.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:00.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:00 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' 2024-09-13T22:44:02.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:02 smithi105 ceph-mon[65554]: pgmap v374: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:02.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:02 smithi118 ceph-mon[51566]: pgmap v374: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:04.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:04 smithi105 ceph-mon[65554]: pgmap v375: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:04.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:04 smithi118 ceph-mon[51566]: pgmap v375: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:06.641 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:06 smithi105 ceph-mon[65554]: pgmap v376: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:06.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:06 smithi118 ceph-mon[51566]: pgmap v376: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:08.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:08 smithi105 ceph-mon[65554]: pgmap v377: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:08.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:08 smithi118 ceph-mon[51566]: pgmap v377: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:10.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:10 smithi118 ceph-mon[51566]: pgmap v378: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:10.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:10 smithi105 ceph-mon[65554]: pgmap v378: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:12.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:12 smithi118 ceph-mon[51566]: pgmap v379: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:12.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:12 smithi105 ceph-mon[65554]: pgmap v379: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:14 smithi118 ceph-mon[51566]: pgmap v380: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:14.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:14 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:44:14.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:14 smithi105 ceph-mon[65554]: pgmap v380: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:14.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:14 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:44:16.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:16 smithi118 ceph-mon[51566]: pgmap v381: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:16.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:16 smithi105 ceph-mon[65554]: pgmap v381: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:18.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:18 smithi118 ceph-mon[51566]: pgmap v382: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:18.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:18 smithi105 ceph-mon[65554]: pgmap v382: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:20.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:20 smithi118 ceph-mon[51566]: pgmap v383: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:20.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:20 smithi105 ceph-mon[65554]: pgmap v383: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:22.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:22 smithi118 ceph-mon[51566]: pgmap v384: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:22.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:22 smithi105 ceph-mon[65554]: pgmap v384: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:24.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:24 smithi118 ceph-mon[51566]: pgmap v385: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:24.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:24 smithi105 ceph-mon[65554]: pgmap v385: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:26.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:26 smithi118 ceph-mon[51566]: pgmap v386: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:26.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:26 smithi105 ceph-mon[65554]: pgmap v386: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:28.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:28 smithi118 ceph-mon[51566]: pgmap v387: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:28.890 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:28 smithi105 ceph-mon[65554]: pgmap v387: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:29.295 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch ps' 2024-09-13T22:44:29.599 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:29 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:44:29.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:29 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:44:30.555 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:30 smithi105 ceph-mon[65554]: pgmap v388: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:30.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:30 smithi118 ceph-mon[51566]: pgmap v388: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:31.281 INFO:teuthology.orchestra.run.smithi105.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T22:44:31.281 INFO:teuthology.orchestra.run.smithi105.stdout:alertmanager.smithi105 smithi105 *:9093,9094 running (12m) 98s ago 21m 27.4M - 0.25.0 c8568f914cd2 d2c05610ea9e 2024-09-13T22:44:31.281 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi105 smithi105 running (8m) 98s ago 21m 6891k - 19.1.1-324-gecab7be5 0652ff2b5d78 44c8040772e4 2024-09-13T22:44:31.281 INFO:teuthology.orchestra.run.smithi105.stdout:crash.smithi118 smithi118 running (8m) 2m ago 20m 6895k - 19.1.1-324-gecab7be5 0652ff2b5d78 6e52611c567b 2024-09-13T22:44:31.281 INFO:teuthology.orchestra.run.smithi105.stdout:grafana.smithi105 smithi105 *:3000 running (105s) 98s ago 21m 42.0M - 9.4.12 2bacad6d85d8 76bb146938c4 2024-09-13T22:44:31.281 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi105.ddoxtb smithi105 running (2m) 98s ago 15m 36.7M - 19.1.1-324-gecab7be5 0652ff2b5d78 a36b4be60260 2024-09-13T22:44:31.281 INFO:teuthology.orchestra.run.smithi105.stdout:mds.foofs.smithi118.izbofj smithi118 running (2m) 2m ago 15m 12.8M - 19.1.1-324-gecab7be5 0652ff2b5d78 c42e4784af3e 2024-09-13T22:44:31.281 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi105.swmmea smithi105 *:8443,9283,8765 running (11m) 98s ago 22m 519M - 19.1.1-324-gecab7be5 0652ff2b5d78 b024b0aff480 2024-09-13T22:44:31.281 INFO:teuthology.orchestra.run.smithi105.stdout:mgr.smithi118.rbhtgz smithi118 *:8443,9283,8765 running (10m) 2m ago 20m 443M - 19.1.1-324-gecab7be5 0652ff2b5d78 4ddfe3ffcb30 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi105 smithi105 running (10m) 98s ago 22m 63.7M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 5855b3e31493 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:mon.smithi118 smithi118 running (9m) 2m ago 20m 48.9M 2048M 19.1.1-324-gecab7be5 0652ff2b5d78 e2c0838729bc 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.0.0.smithi105.xxfcdx smithi105 *:2049 running (2m) 98s ago 15m 60.3M - 5.5 0652ff2b5d78 b6b7bc689a93 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:nfs.foo.1.0.smithi118.chlqvt smithi118 *:2049 running (2m) 2m ago 15m 17.5M - 5.5 0652ff2b5d78 a247799bcbba 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi105 smithi105 *:9100 running (12m) 98s ago 21m 15.4M - 1.5.0 0da6a335fe13 f4a5a3cf434f 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:node-exporter.smithi118 smithi118 *:9100 running (11m) 2m ago 20m 15.5M - 1.5.0 0da6a335fe13 a4ab593d5313 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:osd.0 smithi105 running (8m) 98s ago 20m 396M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 d259261fb822 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:osd.1 smithi105 running (7m) 98s ago 19m 328M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 c70af9930f80 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:osd.2 smithi105 running (6m) 98s ago 19m 369M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 71d2466dd952 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:osd.3 smithi105 running (5m) 98s ago 18m 402M 1936M 19.1.1-324-gecab7be5 0652ff2b5d78 3359b8a2d073 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:osd.4 smithi118 running (5m) 2m ago 18m 368M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 c8d592c38cdd 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:osd.5 smithi118 running (4m) 2m ago 17m 333M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 f03e887cfb9e 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:osd.6 smithi118 running (3m) 2m ago 17m 281M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 e3a4ac1b8c0c 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:osd.7 smithi118 running (3m) 2m ago 17m 302M 2704M 19.1.1-324-gecab7be5 0652ff2b5d78 baaa3d182756 2024-09-13T22:44:31.282 INFO:teuthology.orchestra.run.smithi105.stdout:prometheus.smithi105 smithi105 *:9095 running (10m) 98s ago 20m 62.6M - 2.43.0 a07b618ecd1d 15e50faaf8cb 2024-09-13T22:44:31.938 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph orch upgrade status' 2024-09-13T22:44:32.491 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:32 smithi105 ceph-mon[65554]: pgmap v389: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:32.491 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:32 smithi105 ceph-mon[65554]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:44:32.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:32 smithi118 ceph-mon[51566]: pgmap v389: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:32.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:32 smithi118 ceph-mon[51566]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:44:33.896 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:44:33.896 INFO:teuthology.orchestra.run.smithi105.stdout: "target_image": null, 2024-09-13T22:44:33.896 INFO:teuthology.orchestra.run.smithi105.stdout: "in_progress": false, 2024-09-13T22:44:33.896 INFO:teuthology.orchestra.run.smithi105.stdout: "which": "", 2024-09-13T22:44:33.896 INFO:teuthology.orchestra.run.smithi105.stdout: "services_complete": [], 2024-09-13T22:44:33.897 INFO:teuthology.orchestra.run.smithi105.stdout: "progress": null, 2024-09-13T22:44:33.897 INFO:teuthology.orchestra.run.smithi105.stdout: "message": "", 2024-09-13T22:44:33.897 INFO:teuthology.orchestra.run.smithi105.stdout: "is_paused": false 2024-09-13T22:44:33.897 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:44:34.467 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph health detail' 2024-09-13T22:44:34.492 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:34 smithi105 ceph-mon[65554]: pgmap v390: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:34.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:34 smithi118 ceph-mon[51566]: pgmap v390: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:35.742 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:35 smithi105 ceph-mon[65554]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:44:35.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:35 smithi118 ceph-mon[51566]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:44:36.531 INFO:teuthology.orchestra.run.smithi105.stdout:HEALTH_OK 2024-09-13T22:44:36.833 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:36 smithi105 ceph-mon[65554]: pgmap v391: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:36.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:36 smithi118 ceph-mon[51566]: pgmap v391: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:37.175 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions' 2024-09-13T22:44:37.769 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:37 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/4176035622' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:44:37.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:37 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/4176035622' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T22:44:38.806 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:38 smithi105 ceph-mon[65554]: pgmap v392: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:38.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:38 smithi118 ceph-mon[51566]: pgmap v392: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:39.318 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-13T22:44:39.318 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-13T22:44:39.318 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:44:39.318 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:44:39.318 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-13T22:44:39.318 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:44:39.318 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:44:39.318 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-13T22:44:39.318 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 8 2024-09-13T22:44:39.319 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:44:39.319 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-09-13T22:44:39.319 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 2 2024-09-13T22:44:39.319 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:44:39.319 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-13T22:44:39.319 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)": 14 2024-09-13T22:44:39.319 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:44:39.319 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-13T22:44:39.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:39 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/227925001' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:44:39.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:39 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/227925001' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:44:40.023 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-13T22:44:40.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:40 smithi105 ceph-mon[65554]: pgmap v393: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:40.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:40 smithi118 ceph-mon[51566]: pgmap v393: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:42.123 INFO:teuthology.orchestra.run.smithi105.stdout:true 2024-09-13T22:44:42.719 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:42 smithi105 ceph-mon[65554]: pgmap v394: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:42.719 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:42 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/46269996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:44:42.744 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -e sha1=ecab7be58963a5bd03910cd73eef66405d7fdd50 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-13T22:44:42.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:42 smithi118 ceph-mon[51566]: pgmap v394: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:42.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:42 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/46269996' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:44:44.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:44 smithi105 ceph-mon[65554]: pgmap v395: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:44.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:44 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:44:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:44 smithi118 ceph-mon[51566]: pgmap v395: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:44.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:44 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:44:44.913 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.1.1-324-gecab7be5 (ecab7be58963a5bd03910cd73eef66405d7fdd50) squid (stable)" 2024-09-13T22:44:45.511 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:45 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/459421986' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:44:45.534 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-13T22:44:45.534 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-13T22:44:45.535 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T22:44:45.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:45 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/459421986' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T22:44:46.840 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:46 smithi105 ceph-mon[65554]: pgmap v396: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:46.846 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:46 smithi118 ceph-mon[51566]: pgmap v396: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:47.536 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-13T22:44:47.536 INFO:teuthology.orchestra.run.smithi105.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T22:22:24.273291Z", "last_refresh": "2024-09-13T22:42:52.320145Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T22:22:18.040786Z", "last_refresh": "2024-09-13T22:42:26.681405Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-13T22:22:20.947057Z", "last_refresh": "2024-09-13T22:42:52.320296Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T22:28:57.713669Z", "last_refresh": "2024-09-13T22:42:26.681989Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T22:22:15.908528Z", "last_refresh": "2024-09-13T22:42:26.681501Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi105:172.21.15.105=smithi105", "smithi118:172.21.15.118=smithi118"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T22:23:08.132071Z", "last_refresh": "2024-09-13T22:42:26.681577Z", "running": 2, "size": 2}}, {"events": ["2024-09-13T22:43:59.767955Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-13T22:29:15.679660Z", "last_refresh": "2024-09-13T22:42:26.682056Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T22:22:22.269727Z", "last_refresh": "2024-09-13T22:42:26.681646Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808", "last_refresh": "2024-09-13T22:42:26.681714Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T22:22:19.465524Z", "last_refresh": "2024-09-13T22:42:52.320429Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T22:44:48.186 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-13T22:44:48.186 DEBUG:teuthology.parallel:result is None 2024-09-13T22:44:48.186 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T22:44:48.197 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:44:48.197 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-13T22:44:48.222 INFO:teuthology.orchestra.run.smithi105.stderr:+ umount /mnt/foo 2024-09-13T22:44:48.248 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T22:44:48.258 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:44:48.258 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-13T22:44:48.808 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:48 smithi105 ceph-mon[65554]: pgmap v397: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:48.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:48 smithi118 ceph-mon[51566]: pgmap v397: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:49.554 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:49 smithi105 ceph-mon[65554]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:44:49.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:49 smithi118 ceph-mon[51566]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T22:44:50.313 INFO:teuthology.orchestra.run.smithi105.stdout: "foo" 2024-09-13T22:44:50.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:50 smithi105 ceph-mon[65554]: pgmap v398: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:50.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:50 smithi118 ceph-mon[51566]: pgmap v398: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:51.144 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-13T22:44:51.790 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:51 smithi105 ceph-mon[65554]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:44:51.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:51 smithi118 ceph-mon[51566]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:44:52.696 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:52 smithi105 ceph-mon[65554]: pgmap v399: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:52.748 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:52 smithi118 ceph-mon[51566]: pgmap v399: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:53.168 INFO:teuthology.orchestra.run.smithi105.stdout:[ 2024-09-13T22:44:53.168 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-13T22:44:53.168 INFO:teuthology.orchestra.run.smithi105.stdout: "access_type": "RW", 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: "clients": [], 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_id": "foo", 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: "export_id": 1, 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: "fsal": { 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: "fs_name": "foofs", 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: "name": "CEPH", 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: "user_id": "nfs.foo.1" 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: "path": "/", 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: "protocols": [ 2024-09-13T22:44:53.169 INFO:teuthology.orchestra.run.smithi105.stdout: 4 2024-09-13T22:44:53.170 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-13T22:44:53.170 INFO:teuthology.orchestra.run.smithi105.stdout: "pseudo": "/fake", 2024-09-13T22:44:53.170 INFO:teuthology.orchestra.run.smithi105.stdout: "security_label": true, 2024-09-13T22:44:53.170 INFO:teuthology.orchestra.run.smithi105.stdout: "squash": "none", 2024-09-13T22:44:53.170 INFO:teuthology.orchestra.run.smithi105.stdout: "transports": [ 2024-09-13T22:44:53.170 INFO:teuthology.orchestra.run.smithi105.stdout: "TCP" 2024-09-13T22:44:53.170 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-13T22:44:53.170 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-13T22:44:53.170 INFO:teuthology.orchestra.run.smithi105.stdout:] 2024-09-13T22:44:53.864 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-13T22:44:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:54 smithi118 ceph-mon[51566]: pgmap v400: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:54.848 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:54 smithi118 ceph-mon[51566]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:44:54.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:54 smithi105 ceph-mon[65554]: pgmap v400: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T22:44:54.858 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:54 smithi105 ceph-mon[65554]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T22:44:55.644 INFO:teuthology.orchestra.run.smithi105.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-13T22:44:55.644 INFO:teuthology.orchestra.run.smithi105.stdout:foo grace 2024-09-13T22:44:55.644 INFO:teuthology.orchestra.run.smithi105.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-13T22:44:55.644 INFO:teuthology.orchestra.run.smithi105.stdout:foo conf-nfs.foo 2024-09-13T22:44:55.644 INFO:teuthology.orchestra.run.smithi105.stdout:foo export-1 2024-09-13T22:44:56.258 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T22:44:56.270 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:44:56.270 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-13T22:44:56.270 DEBUG:teuthology.orchestra.run.smithi105:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-13T22:44:56.270 DEBUG:teuthology.orchestra.run.smithi105:> ' 2024-09-13T22:44:56.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:56 smithi105 ceph-mon[65554]: pgmap v401: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:56.843 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:56 smithi105 ceph-mon[65554]: mgrmap e44: smithi105.swmmea(active, since 10m), standbys: smithi118.rbhtgz 2024-09-13T22:44:56.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:56 smithi118 ceph-mon[51566]: pgmap v401: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T22:44:56.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:56 smithi118 ceph-mon[51566]: mgrmap e44: smithi105.swmmea(active, since 10m), standbys: smithi118.rbhtgz 2024-09-13T22:44:58.099 INFO:teuthology.orchestra.run.smithi105.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-13T22:44:58.326 INFO:teuthology.orchestra.run.smithi105.stderr:+ [[ 7 -gt 2 ]] 2024-09-13T22:44:58.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:58 smithi105 ceph-mon[65554]: pgmap v402: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T22:44:58.640 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:58 smithi105 ceph-mon[65554]: from='client.? 172.21.15.105:0/2793976677' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-13T22:44:58.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:58 smithi118 ceph-mon[51566]: pgmap v402: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T22:44:58.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:58 smithi118 ceph-mon[51566]: from='client.? 172.21.15.105:0/2793976677' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-13T22:44:59.069 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T22:44:59.080 INFO:tasks.cephadm:Teardown begin 2024-09-13T22:44:59.080 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:44:59.109 DEBUG:teuthology.orchestra.run.smithi118:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:44:59.143 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T22:44:59.144 DEBUG:teuthology.orchestra.run.smithi105:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T22:44:59.164 DEBUG:teuthology.orchestra.run.smithi118:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T22:44:59.202 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T22:44:59.203 INFO:tasks.cephadm.mon.smithi105:Stopping mon.smithi105... 2024-09-13T22:44:59.203 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi105 2024-09-13T22:44:59.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:59 smithi105 systemd[1]: Stopping Ceph mon.smithi105 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:44:59.754 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:59 smithi105 ceph-mon[65554]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:44:59.754 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:59 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105[65550]: 2024-09-13T22:44:59.498+0000 7f24fc0ef640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi105 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:44:59.754 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:59 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105[65550]: 2024-09-13T22:44:59.498+0000 7f24fc0ef640 -1 mon.smithi105@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T22:44:59.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:44:59 smithi118 ceph-mon[51566]: from='mgr.34108 172.21.15.105:0/408005948' entity='mgr.smithi105.swmmea' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T22:45:00.058 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:44:59 smithi105 podman[99013]: 2024-09-13 22:44:59.75392472 +0000 UTC m=+0.363360463 container died 5855b3e3149363c1d95a2e13c82fd5515d0cc5f65e215e7eff28ef2a6907052d (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:45:00.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:45:00 smithi105 podman[99013]: 2024-09-13 22:45:00.165292426 +0000 UTC m=+0.774728167 container cleanup 5855b3e3149363c1d95a2e13c82fd5515d0cc5f65e215e7eff28ef2a6907052d (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T22:45:00.390 INFO:journalctl@ceph.mon.smithi105.smithi105.stdout:Sep 13 22:45:00 smithi105 bash[99013]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi105 2024-09-13T22:45:00.579 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi105.service' 2024-09-13T22:45:00.625 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:45:00.625 INFO:tasks.cephadm.mon.smithi105:Stopped mon.smithi105 2024-09-13T22:45:00.625 INFO:tasks.cephadm.mon.smithi118:Stopping mon.smithi118... 2024-09-13T22:45:00.625 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi118 2024-09-13T22:45:00.961 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:45:00 smithi118 systemd[1]: Stopping Ceph mon.smithi118 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:45:01.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:45:00 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118[51562]: 2024-09-13T22:45:00.959+0000 7f2b04a28640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi118 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:45:01.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:45:00 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118[51562]: 2024-09-13T22:45:00.959+0000 7f2b04a28640 -1 mon.smithi118@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T22:45:01.347 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:45:01 smithi118 podman[73023]: 2024-09-13 22:45:01.113301303 +0000 UTC m=+0.293695635 container died e2c0838729bcd956b282ca84a09ac0a04820a7f5ffb2d8d3d9311908606f1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True) 2024-09-13T22:45:01.836 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:45:01 smithi118 podman[73023]: 2024-09-13 22:45:01.559757071 +0000 UTC m=+0.740151400 container cleanup e2c0838729bcd956b282ca84a09ac0a04820a7f5ffb2d8d3d9311908606f1b43 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T22:45:01.836 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 13 22:45:01 smithi118 bash[73023]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-mon-smithi118 2024-09-13T22:45:01.985 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@mon.smithi118.service' 2024-09-13T22:45:02.027 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:45:02.028 INFO:tasks.cephadm.mon.smithi118:Stopped mon.smithi118 2024-09-13T22:45:02.028 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T22:45:02.028 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.0 2024-09-13T22:45:02.320 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:02 smithi105 systemd[1]: Stopping Ceph osd.0 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:45:02.640 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:02 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[71707]: 2024-09-13T22:45:02.319+0000 7fd63405d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:45:02.640 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:02 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[71707]: 2024-09-13T22:45:02.319+0000 7fd63405d640 -1 osd.0 159 *** Got signal Terminated *** 2024-09-13T22:45:02.640 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:02 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0[71707]: 2024-09-13T22:45:02.319+0000 7fd63405d640 -1 osd.0 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:45:08.141 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:07 smithi105 podman[99178]: 2024-09-13 22:45:07.92186464 +0000 UTC m=+5.702449502 container died d259261fb822184f8dd37566365c660016e6de12589576eea60835b56f33a580 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T22:45:08.582 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:08 smithi105 podman[99178]: 2024-09-13 22:45:08.305790783 +0000 UTC m=+6.086375632 container cleanup d259261fb822184f8dd37566365c660016e6de12589576eea60835b56f33a580 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:45:08.582 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:08 smithi105 bash[99178]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0 2024-09-13T22:45:08.890 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:08 smithi105 podman[99193]: 2024-09-13 22:45:08.581728105 +0000 UTC m=+0.649692101 container remove d259261fb822184f8dd37566365c660016e6de12589576eea60835b56f33a580 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-ecab7be, ceph=True) 2024-09-13T22:45:09.202 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:08 smithi105 podman[99282]: 2024-09-13 22:45:08.827741591 +0000 UTC m=+0.015768761 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:45:09.202 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:08 smithi105 podman[99282]: 2024-09-13 22:45:08.957689964 +0000 UTC m=+0.145717129 container create bf3b1b0d3e918403ba1bf1a992e684a6e7b83e08ef0f1b5a72feef0a3b9132e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:45:09.202 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:09 smithi105 podman[99282]: 2024-09-13 22:45:09.201835 +0000 UTC m=+0.389862185 container init bf3b1b0d3e918403ba1bf1a992e684a6e7b83e08ef0f1b5a72feef0a3b9132e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T22:45:09.640 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:09 smithi105 podman[99282]: 2024-09-13 22:45:09.205186822 +0000 UTC m=+0.393213991 container start bf3b1b0d3e918403ba1bf1a992e684a6e7b83e08ef0f1b5a72feef0a3b9132e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T22:45:09.640 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:09 smithi105 podman[99282]: 2024-09-13 22:45:09.293614696 +0000 UTC m=+0.481641874 container attach bf3b1b0d3e918403ba1bf1a992e684a6e7b83e08ef0f1b5a72feef0a3b9132e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-13T22:45:09.640 INFO:journalctl@ceph.osd.0.smithi105.stdout:Sep 13 22:45:09 smithi105 podman[99282]: 2024-09-13 22:45:09.397676958 +0000 UTC m=+0.585704128 container died bf3b1b0d3e918403ba1bf1a992e684a6e7b83e08ef0f1b5a72feef0a3b9132e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T22:45:10.013 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T22:45:10.053 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:45:10.053 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T22:45:10.054 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T22:45:10.054 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.1 2024-09-13T22:45:10.384 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:10 smithi105 systemd[1]: Stopping Ceph osd.1 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:45:10.854 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:10 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[74299]: 2024-09-13T22:45:10.383+0000 7fe7ca37b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:45:10.855 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:10 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[74299]: 2024-09-13T22:45:10.383+0000 7fe7ca37b640 -1 osd.1 159 *** Got signal Terminated *** 2024-09-13T22:45:10.855 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:10 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1[74299]: 2024-09-13T22:45:10.383+0000 7fe7ca37b640 -1 osd.1 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:45:16.268 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:15 smithi105 podman[99385]: 2024-09-13 22:45:15.970707149 +0000 UTC m=+5.691691233 container died c70af9930f804bbc4fc8e21bfcb635a6abe60d428d6785e65308a3e3ad3d422b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:45:16.610 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:16 smithi105 podman[99385]: 2024-09-13 22:45:16.333351539 +0000 UTC m=+6.054335613 container cleanup c70af9930f804bbc4fc8e21bfcb635a6abe60d428d6785e65308a3e3ad3d422b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:45:16.610 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:16 smithi105 bash[99385]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1 2024-09-13T22:45:16.890 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:16 smithi105 podman[99401]: 2024-09-13 22:45:16.609709314 +0000 UTC m=+0.640110136 container remove c70af9930f804bbc4fc8e21bfcb635a6abe60d428d6785e65308a3e3ad3d422b (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=) 2024-09-13T22:45:17.213 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:16 smithi105 podman[99488]: 2024-09-13 22:45:16.852501117 +0000 UTC m=+0.016356777 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:45:17.213 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:16 smithi105 podman[99488]: 2024-09-13 22:45:16.960609667 +0000 UTC m=+0.124465366 container create 02e0b8cf6917630588e1cf59e19072c4c4dcbbd092589422c9e2859170e4c7b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be) 2024-09-13T22:45:17.213 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:17 smithi105 podman[99488]: 2024-09-13 22:45:17.213272947 +0000 UTC m=+0.377128610 container init 02e0b8cf6917630588e1cf59e19072c4c4dcbbd092589422c9e2859170e4c7b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T22:45:17.640 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:17 smithi105 podman[99488]: 2024-09-13 22:45:17.216819797 +0000 UTC m=+0.380675457 container start 02e0b8cf6917630588e1cf59e19072c4c4dcbbd092589422c9e2859170e4c7b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T22:45:17.640 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:17 smithi105 podman[99488]: 2024-09-13 22:45:17.313406048 +0000 UTC m=+0.477261722 container attach 02e0b8cf6917630588e1cf59e19072c4c4dcbbd092589422c9e2859170e4c7b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS) 2024-09-13T22:45:17.640 INFO:journalctl@ceph.osd.1.smithi105.stdout:Sep 13 22:45:17 smithi105 podman[99488]: 2024-09-13 22:45:17.410694295 +0000 UTC m=+0.574549950 container died 02e0b8cf6917630588e1cf59e19072c4c4dcbbd092589422c9e2859170e4c7b6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-13T22:45:18.003 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T22:45:18.041 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:45:18.042 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T22:45:18.042 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T22:45:18.042 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.2 2024-09-13T22:45:18.387 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:18 smithi105 systemd[1]: Stopping Ceph osd.2 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:45:18.640 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:18 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[76945]: 2024-09-13T22:45:18.386+0000 7f01c2b3b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:45:18.640 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:18 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[76945]: 2024-09-13T22:45:18.386+0000 7f01c2b3b640 -1 osd.2 159 *** Got signal Terminated *** 2024-09-13T22:45:18.640 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:18 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2[76945]: 2024-09-13T22:45:18.386+0000 7f01c2b3b640 -1 osd.2 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:45:24.080 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:23 smithi105 podman[99591]: 2024-09-13 22:45:23.794760649 +0000 UTC m=+5.525086263 container died 71d2466dd952bd2c2357bd204776e5ac3503b3e557889aa65ebb9e8259e8b696 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T22:45:24.390 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:24 smithi105 podman[99591]: 2024-09-13 22:45:24.178279201 +0000 UTC m=+5.908604818 container cleanup 71d2466dd952bd2c2357bd204776e5ac3503b3e557889aa65ebb9e8259e8b696 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:45:24.390 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:24 smithi105 bash[99591]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2 2024-09-13T22:45:24.796 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:24 smithi105 podman[99638]: 2024-09-13 22:45:24.45493108 +0000 UTC m=+0.661663339 container remove 71d2466dd952bd2c2357bd204776e5ac3503b3e557889aa65ebb9e8259e8b696 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T22:45:25.133 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:24 smithi105 podman[99727]: 2024-09-13 22:45:24.699195583 +0000 UTC m=+0.016362386 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:45:25.133 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:24 smithi105 podman[99727]: 2024-09-13 22:45:24.822599553 +0000 UTC m=+0.139766332 container create 345b5c17c533dce15a768149731253d5edb99284db194d0d3f1197a04399f72f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T22:45:25.133 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:25 smithi105 podman[99727]: 2024-09-13 22:45:25.03335023 +0000 UTC m=+0.350517015 container init 345b5c17c533dce15a768149731253d5edb99284db194d0d3f1197a04399f72f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T22:45:25.134 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:25 smithi105 podman[99727]: 2024-09-13 22:45:25.036644745 +0000 UTC m=+0.353811525 container start 345b5c17c533dce15a768149731253d5edb99284db194d0d3f1197a04399f72f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T22:45:25.134 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:25 smithi105 podman[99727]: 2024-09-13 22:45:25.133236778 +0000 UTC m=+0.450403570 container attach 345b5c17c533dce15a768149731253d5edb99284db194d0d3f1197a04399f72f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2) 2024-09-13T22:45:25.390 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:25 smithi105 podman[99727]: 2024-09-13 22:45:25.236340759 +0000 UTC m=+0.553507551 container died 345b5c17c533dce15a768149731253d5edb99284db194d0d3f1197a04399f72f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T22:45:25.782 INFO:journalctl@ceph.osd.2.smithi105.stdout:Sep 13 22:45:25 smithi105 podman[99750]: 2024-09-13 22:45:25.778670602 +0000 UTC m=+0.535787122 container remove 345b5c17c533dce15a768149731253d5edb99284db194d0d3f1197a04399f72f (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T22:45:25.801 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T22:45:25.840 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:45:25.840 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T22:45:25.840 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T22:45:25.840 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.3 2024-09-13T22:45:26.140 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:25 smithi105 systemd[1]: Stopping Ceph osd.3 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:45:26.640 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:26 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[79635]: 2024-09-13T22:45:26.177+0000 7fd1ad117640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:45:26.640 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:26 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[79635]: 2024-09-13T22:45:26.177+0000 7fd1ad117640 -1 osd.3 159 *** Got signal Terminated *** 2024-09-13T22:45:26.640 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:26 smithi105 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3[79635]: 2024-09-13T22:45:26.177+0000 7fd1ad117640 -1 osd.3 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:45:28.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:28 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:28.367+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:29.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:28 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:28.935+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:29.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:29 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:29.390+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:30.300 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:29 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:29.925+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:30.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:30 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:30.357+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:31.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:31 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:31.142+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:31.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:30 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:30.934+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:31.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:31 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:31.384+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:31.874 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:31 smithi105 podman[99830]: 2024-09-13 22:45:31.584247328 +0000 UTC m=+5.521903796 container died 3359b8a2d07315f2c811916b1d1d9730ab7b20e7ba2a4d73597373588d0adf23 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, ceph=True, org.label-schema.vendor=CentOS) 2024-09-13T22:45:32.140 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:31 smithi105 podman[99830]: 2024-09-13 22:45:31.956361598 +0000 UTC m=+5.894018055 container cleanup 3359b8a2d07315f2c811916b1d1d9730ab7b20e7ba2a4d73597373588d0adf23 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-13T22:45:32.140 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:31 smithi105 bash[99830]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3 2024-09-13T22:45:32.192 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:31 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:31.937+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:32.568 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:32 smithi105 podman[99848]: 2024-09-13 22:45:32.240952827 +0000 UTC m=+0.651537104 container remove 3359b8a2d07315f2c811916b1d1d9730ab7b20e7ba2a4d73597373588d0adf23 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T22:45:32.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:32 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:32.432+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:32.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:32 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:32.191+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:32.890 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:32 smithi105 podman[99936]: 2024-09-13 22:45:32.470279831 +0000 UTC m=+0.016081589 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:45:32.891 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:32 smithi105 podman[99936]: 2024-09-13 22:45:32.591909836 +0000 UTC m=+0.137711646 container create 151bc30aa26b4db04ca4aa044efe8e462e6e8de4562f8a3eca5c4ed78077b2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be) 2024-09-13T22:45:32.891 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:32 smithi105 podman[99936]: 2024-09-13 22:45:32.81091101 +0000 UTC m=+0.356712759 container init 151bc30aa26b4db04ca4aa044efe8e462e6e8de4562f8a3eca5c4ed78077b2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T22:45:32.891 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:32 smithi105 podman[99936]: 2024-09-13 22:45:32.814412158 +0000 UTC m=+0.360213910 container start 151bc30aa26b4db04ca4aa044efe8e462e6e8de4562f8a3eca5c4ed78077b2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T22:45:33.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:33 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:33.142+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:33.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:32 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:32.928+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:33.390 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:32 smithi105 podman[99936]: 2024-09-13 22:45:32.894293896 +0000 UTC m=+0.440095645 container attach 151bc30aa26b4db04ca4aa044efe8e462e6e8de4562f8a3eca5c4ed78077b2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T22:45:33.390 INFO:journalctl@ceph.osd.3.smithi105.stdout:Sep 13 22:45:33 smithi105 podman[99936]: 2024-09-13 22:45:33.004042711 +0000 UTC m=+0.549844460 container died 151bc30aa26b4db04ca4aa044efe8e462e6e8de4562f8a3eca5c4ed78077b2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-13T22:45:33.717 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T22:45:33.757 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:45:33.758 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T22:45:33.758 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T22:45:33.758 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.4 2024-09-13T22:45:33.786 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:33 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:33.437+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:33.787 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:33 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:33.468+0000 7f5de5d28640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:04.350014+0000 front 2024-09-13T22:45:04.350165+0000 (oldest deadline 2024-09-13T22:45:33.150322+0000) 2024-09-13T22:45:34.062 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:33 smithi118 systemd[1]: Stopping Ceph osd.4 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:45:34.062 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:33 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:33.967+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:34.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:34.165+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:34.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:34.061+0000 7f5ded71a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:45:34.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:34.061+0000 7f5ded71a640 -1 osd.4 159 *** Got signal Terminated *** 2024-09-13T22:45:34.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:34.061+0000 7f5ded71a640 -1 osd.4 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:45:34.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:34.515+0000 7f5de5d28640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:04.350014+0000 front 2024-09-13T22:45:04.350165+0000 (oldest deadline 2024-09-13T22:45:33.150322+0000) 2024-09-13T22:45:34.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:34.464+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:35.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:35 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:35.205+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:35.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:34 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:34.971+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:35.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:35 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:35.495+0000 7f5de5d28640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:04.350014+0000 front 2024-09-13T22:45:04.350165+0000 (oldest deadline 2024-09-13T22:45:33.150322+0000) 2024-09-13T22:45:35.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:35 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:35.492+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:36.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:35 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:35.951+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:36.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:36 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:36.168+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:36.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:36 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:36.505+0000 7f5de5d28640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:04.350014+0000 front 2024-09-13T22:45:04.350165+0000 (oldest deadline 2024-09-13T22:45:33.150322+0000) 2024-09-13T22:45:36.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:36 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:36.505+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:36.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:36 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:36.505+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:37.178 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:36 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:36.907+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:37.476 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:37 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:37.177+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:37.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:37 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:37.540+0000 7f5de5d28640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:04.350014+0000 front 2024-09-13T22:45:04.350165+0000 (oldest deadline 2024-09-13T22:45:33.150322+0000) 2024-09-13T22:45:37.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:37 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:37.540+0000 7f5de5d28640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.151122+0000 front 2024-09-13T22:45:13.151155+0000 (oldest deadline 2024-09-13T22:45:36.650899+0000) 2024-09-13T22:45:37.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:37 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:37.475+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:37.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:37 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:37.475+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:38.183 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:37 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:37.886+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:38.446 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:38 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:38.182+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:38.446 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:38 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:38.182+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:38.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:38 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:38.568+0000 7f5de5d28640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:04.350014+0000 front 2024-09-13T22:45:04.350165+0000 (oldest deadline 2024-09-13T22:45:33.150322+0000) 2024-09-13T22:45:38.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:38 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4[56410]: 2024-09-13T22:45:38.568+0000 7f5de5d28640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.151122+0000 front 2024-09-13T22:45:13.151155+0000 (oldest deadline 2024-09-13T22:45:36.650899+0000) 2024-09-13T22:45:38.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:38 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:38.445+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:38.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:38 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:38.445+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:39.193 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:38 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:38.852+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:39.468 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:39 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:39.192+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:39.468 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:39 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:39.192+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:39.847 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:39 smithi118 podman[73223]: 2024-09-13 22:45:39.628413185 +0000 UTC m=+5.668336904 container died c8d592c38cddb153d150e7282080ae7d93e38a8b1fc891d1c862868fe631bf96 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T22:45:39.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:39 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:39.467+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:39.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:39 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:39.467+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:40.224 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:39 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:39.901+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:40.224 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:40 smithi118 podman[73223]: 2024-09-13 22:45:40.069802555 +0000 UTC m=+6.109726271 container cleanup c8d592c38cddb153d150e7282080ae7d93e38a8b1fc891d1c862868fe631bf96 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be) 2024-09-13T22:45:40.224 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:40 smithi118 bash[73223]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4 2024-09-13T22:45:40.485 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:40 smithi118 podman[73238]: 2024-09-13 22:45:40.262791867 +0000 UTC m=+0.633498916 container remove c8d592c38cddb153d150e7282080ae7d93e38a8b1fc891d1c862868fe631bf96 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-13T22:45:40.485 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:40 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:40.222+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:40.485 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:40 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:40.222+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:40.804 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:40 smithi118 podman[73327]: 2024-09-13 22:45:40.583444292 +0000 UTC m=+0.015734506 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:45:40.805 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:40 smithi118 podman[73327]: 2024-09-13 22:45:40.697058191 +0000 UTC m=+0.129348396 container create b827cca46060d9a0510e0d228771ea5c82811904007f56ff94b59617b05d15e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be) 2024-09-13T22:45:40.805 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:40 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:40.484+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:40.805 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:40 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:40.484+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:41.082 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:40 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:40.950+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:41.082 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:40 smithi118 podman[73327]: 2024-09-13 22:45:40.916035421 +0000 UTC m=+0.348325625 container init b827cca46060d9a0510e0d228771ea5c82811904007f56ff94b59617b05d15e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-ecab7be, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T22:45:41.082 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:40 smithi118 podman[73327]: 2024-09-13 22:45:40.919296046 +0000 UTC m=+0.351586251 container start b827cca46060d9a0510e0d228771ea5c82811904007f56ff94b59617b05d15e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-13T22:45:41.083 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:41 smithi118 podman[73327]: 2024-09-13 22:45:41.007679753 +0000 UTC m=+0.439969971 container attach b827cca46060d9a0510e0d228771ea5c82811904007f56ff94b59617b05d15e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T22:45:41.347 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 13 22:45:41 smithi118 podman[73327]: 2024-09-13 22:45:41.107698993 +0000 UTC m=+0.539989198 container died b827cca46060d9a0510e0d228771ea5c82811904007f56ff94b59617b05d15e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-ecab7be) 2024-09-13T22:45:41.631 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:41 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:41.208+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:41.632 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:41 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:41.208+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:41.746 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:41 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:41.466+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:41.746 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:41 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:41.466+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:41.765 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T22:45:41.847 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:45:41.847 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T22:45:41.847 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T22:45:41.848 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.5 2024-09-13T22:45:42.097 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:41 smithi118 systemd[1]: Stopping Ceph osd.5 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:45:42.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:41 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:41.954+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:42.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:41 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:41.954+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:42.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:42 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:42.200+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:42.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:42 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:42.200+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:42.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:42 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:42.206+0000 7fb4d86d3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:45:42.598 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:42 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:42.206+0000 7fb4d86d3640 -1 osd.5 159 *** Got signal Terminated *** 2024-09-13T22:45:42.598 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:42 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:42.206+0000 7fb4d86d3640 -1 osd.5 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:45:42.598 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:42 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:42.427+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:42.598 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:42 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:42.427+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:43.200 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:42 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:42.908+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:43.200 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:42 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:42.908+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:43.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:43 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:43.415+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:43.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:43 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:43.415+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:43.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:43 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:43.199+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:43.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:43 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:43.199+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:44.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:43 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:43.957+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:44.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:43 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:43.957+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:44.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:44 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:44.168+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:44.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:44 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:44.168+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:44.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:44 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:44.419+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:44.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:44 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:44.419+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:45.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:44 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:44.933+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:45.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:44 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:44.933+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:45.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:45.178+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:45.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:45.178+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:45.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:45.422+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:45.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:45.422+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:45.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:45.422+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:19.293882+0000 front 2024-09-13T22:45:19.293825+0000 (oldest deadline 2024-09-13T22:45:44.593778+0000) 2024-09-13T22:45:46.166 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:45.889+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:46.166 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:45 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:45.889+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:46.429 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:46 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:46.165+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:46.430 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:46 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:46.165+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:46.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:46 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:46.428+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:02.391522+0000 front 2024-09-13T22:45:02.391638+0000 (oldest deadline 2024-09-13T22:45:28.191964+0000) 2024-09-13T22:45:46.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:46 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:46.428+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:13.493017+0000 front 2024-09-13T22:45:13.493030+0000 (oldest deadline 2024-09-13T22:45:36.392846+0000) 2024-09-13T22:45:46.847 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:46 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5[58664]: 2024-09-13T22:45:46.428+0000 7fb4d0ce1640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:19.293882+0000 front 2024-09-13T22:45:19.293825+0000 (oldest deadline 2024-09-13T22:45:44.593778+0000) 2024-09-13T22:45:47.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:47 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:47.141+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:47.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:47 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:47.141+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:47.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:46 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:46.895+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:47.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:46 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:46.895+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:48.015 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:47 smithi118 podman[73474]: 2024-09-13 22:45:47.72900807 +0000 UTC m=+5.641847056 container died f03e887cfb9eb0fc1d3c91b69a5eac729960f4ef90624281644d74701f1fb6e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-13T22:45:48.015 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:47 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:47.891+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:48.016 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:47 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:47.891+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:48.016 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:47 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:47.891+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:48.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:48 smithi118 podman[73474]: 2024-09-13 22:45:48.181285578 +0000 UTC m=+6.094124566 container cleanup f03e887cfb9eb0fc1d3c91b69a5eac729960f4ef90624281644d74701f1fb6e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T22:45:48.347 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:48 smithi118 bash[73474]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5 2024-09-13T22:45:48.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:48 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:48.135+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:48.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:48 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:48.135+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:48.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:48 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:48.135+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:17.873249+0000 front 2024-09-13T22:45:17.873286+0000 (oldest deadline 2024-09-13T22:45:47.273721+0000) 2024-09-13T22:45:48.799 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:48 smithi118 podman[73490]: 2024-09-13 22:45:48.474679961 +0000 UTC m=+0.745397445 container remove f03e887cfb9eb0fc1d3c91b69a5eac729960f4ef90624281644d74701f1fb6e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-ecab7be, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T22:45:49.097 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:48 smithi118 podman[73580]: 2024-09-13 22:45:48.701354611 +0000 UTC m=+0.015721749 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:45:49.097 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:48 smithi118 podman[73580]: 2024-09-13 22:45:48.817069761 +0000 UTC m=+0.131436890 container create c14f3255729a5047ef5e592cabf65c94c636ed59e5c4a07ce0ebc732b6149875 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T22:45:49.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:48 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:48.852+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:49.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:48 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:48.852+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:49.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:48 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:48.852+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:49.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:49 smithi118 podman[73580]: 2024-09-13 22:45:49.120293533 +0000 UTC m=+0.434660666 container init c14f3255729a5047ef5e592cabf65c94c636ed59e5c4a07ce0ebc732b6149875 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-13T22:45:49.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:49 smithi118 podman[73580]: 2024-09-13 22:45:49.124152219 +0000 UTC m=+0.438519364 container start c14f3255729a5047ef5e592cabf65c94c636ed59e5c4a07ce0ebc732b6149875 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T22:45:49.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:49 smithi118 podman[73580]: 2024-09-13 22:45:49.219631695 +0000 UTC m=+0.533998847 container attach c14f3255729a5047ef5e592cabf65c94c636ed59e5c4a07ce0ebc732b6149875 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-13T22:45:49.597 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 13 22:45:49 smithi118 podman[73580]: 2024-09-13 22:45:49.316137682 +0000 UTC m=+0.630504824 container died c14f3255729a5047ef5e592cabf65c94c636ed59e5c4a07ce0ebc732b6149875 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, RELEASE=squid-ecab7be) 2024-09-13T22:45:49.598 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:49 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:49.157+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:49.598 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:49 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:49.157+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:49.598 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:49 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:49.157+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:17.873249+0000 front 2024-09-13T22:45:17.873286+0000 (oldest deadline 2024-09-13T22:45:47.273721+0000) 2024-09-13T22:45:49.944 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T22:45:49.984 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:45:49.985 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T22:45:49.985 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T22:45:49.985 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.6 2024-09-13T22:45:50.318 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:50 smithi118 systemd[1]: Stopping Ceph osd.6 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:45:50.318 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:50 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:50.136+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:50.319 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:50 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:50.136+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:50.319 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:50 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:50.136+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:17.873249+0000 front 2024-09-13T22:45:17.873286+0000 (oldest deadline 2024-09-13T22:45:47.273721+0000) 2024-09-13T22:45:50.319 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:49 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:49.890+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:50.319 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:49 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:49.890+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:50.319 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:49 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:49.890+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:50.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:50 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:50.317+0000 7fd05b66e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:45:50.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:50 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:50.317+0000 7fd05b66e640 -1 osd.6 159 *** Got signal Terminated *** 2024-09-13T22:45:50.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:50 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:50.317+0000 7fd05b66e640 -1 osd.6 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:45:51.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:51 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:51.106+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:51.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:51 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:51.106+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:51.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:51 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:51.106+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:17.873249+0000 front 2024-09-13T22:45:17.873286+0000 (oldest deadline 2024-09-13T22:45:47.273721+0000) 2024-09-13T22:45:51.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:50 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:50.907+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:51.348 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:50 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:50.907+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:51.348 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:50 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:50.907+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:52.096 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:51 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:51.904+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:52.096 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:51 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:51.904+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:52.096 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:51 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:51.904+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:52.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:52 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:52.095+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:52.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:52 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:52.095+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:52.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:52 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:52.095+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:17.873249+0000 front 2024-09-13T22:45:17.873286+0000 (oldest deadline 2024-09-13T22:45:47.273721+0000) 2024-09-13T22:45:53.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:52 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:52.901+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:53.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:52 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:52.901+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:53.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:52 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:52.901+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:53.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:53 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:53.130+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:53.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:53 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:53.130+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:53.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:53 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:53.130+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:17.873249+0000 front 2024-09-13T22:45:17.873286+0000 (oldest deadline 2024-09-13T22:45:47.273721+0000) 2024-09-13T22:45:53.348 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:53 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:53.130+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.174832+0000 front 2024-09-13T22:45:30.174881+0000 (oldest deadline 2024-09-13T22:45:53.074850+0000) 2024-09-13T22:45:54.143 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:53 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:53.867+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:54.143 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:53 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:53.867+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:54.143 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:53 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:53.867+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:54.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:54 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:54.142+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:54.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:54 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:54.142+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:54.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:54 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:54.142+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:17.873249+0000 front 2024-09-13T22:45:17.873286+0000 (oldest deadline 2024-09-13T22:45:47.273721+0000) 2024-09-13T22:45:54.597 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:54 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:54.142+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.174832+0000 front 2024-09-13T22:45:30.174881+0000 (oldest deadline 2024-09-13T22:45:53.074850+0000) 2024-09-13T22:45:55.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:55.158+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:01.571148+0000 front 2024-09-13T22:45:01.571124+0000 (oldest deadline 2024-09-13T22:45:30.371604+0000) 2024-09-13T22:45:55.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:55.158+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.072637+0000 front 2024-09-13T22:45:15.072529+0000 (oldest deadline 2024-09-13T22:45:37.372343+0000) 2024-09-13T22:45:55.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:55.158+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:17.873249+0000 front 2024-09-13T22:45:17.873286+0000 (oldest deadline 2024-09-13T22:45:47.273721+0000) 2024-09-13T22:45:55.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6[60922]: 2024-09-13T22:45:55.158+0000 7fd057c84640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.174832+0000 front 2024-09-13T22:45:30.174881+0000 (oldest deadline 2024-09-13T22:45:53.074850+0000) 2024-09-13T22:45:55.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:54 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:54.914+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:55.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:54 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:54.914+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:55.348 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:54 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:54.914+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:56.085 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:55 smithi118 podman[73682]: 2024-09-13 22:45:55.816565404 +0000 UTC m=+5.608118548 container died e3a4ac1b8c0c38d160e6ef164903c6ced19b8ff80dac8030f83b006cafcf6cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T22:45:56.085 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:55.887+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:56.085 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:55.887+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:56.085 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:55 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:55.887+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:56.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:56 smithi118 podman[73682]: 2024-09-13 22:45:56.192401763 +0000 UTC m=+5.983954907 container cleanup e3a4ac1b8c0c38d160e6ef164903c6ced19b8ff80dac8030f83b006cafcf6cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T22:45:56.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:56 smithi118 bash[73682]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6 2024-09-13T22:45:56.803 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:56 smithi118 podman[73698]: 2024-09-13 22:45:56.493875092 +0000 UTC m=+0.678877878 container remove e3a4ac1b8c0c38d160e6ef164903c6ced19b8ff80dac8030f83b006cafcf6cb6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T22:45:57.072 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:56 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:56.919+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:57.072 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:56 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:56.919+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:57.072 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:56 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:56.919+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:57.072 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:56 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:56.919+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.523093+0000 front 2024-09-13T22:45:30.523137+0000 (oldest deadline 2024-09-13T22:45:56.423133+0000) 2024-09-13T22:45:57.072 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:56 smithi118 podman[73786]: 2024-09-13 22:45:56.802905399 +0000 UTC m=+0.103241712 container create b05c294fcd5c0f6fe06343d78442b515c749fbe8f079208d5ac6022ffe8743f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T22:45:57.072 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:56 smithi118 podman[73786]: 2024-09-13 22:45:56.7156118 +0000 UTC m=+0.015948076 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:45:57.072 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:56 smithi118 podman[73786]: 2024-09-13 22:45:56.99707573 +0000 UTC m=+0.297412000 container init b05c294fcd5c0f6fe06343d78442b515c749fbe8f079208d5ac6022ffe8743f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T22:45:57.073 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:57 smithi118 podman[73786]: 2024-09-13 22:45:57.000384689 +0000 UTC m=+0.300720973 container start b05c294fcd5c0f6fe06343d78442b515c749fbe8f079208d5ac6022ffe8743f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T22:45:57.073 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:57 smithi118 podman[73786]: 2024-09-13 22:45:57.071745322 +0000 UTC m=+0.372081608 container attach b05c294fcd5c0f6fe06343d78442b515c749fbe8f079208d5ac6022ffe8743f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-ecab7be, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T22:45:57.347 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 13 22:45:57 smithi118 podman[73786]: 2024-09-13 22:45:57.182447954 +0000 UTC m=+0.482784232 container died b05c294fcd5c0f6fe06343d78442b515c749fbe8f079208d5ac6022ffe8743f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T22:45:57.761 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T22:45:57.799 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:45:57.799 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T22:45:57.800 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T22:45:57.800 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.7 2024-09-13T22:45:58.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:57 smithi118 systemd[1]: Stopping Ceph osd.7 for 782bdfc8-721e-11ef-bceb-c7b262605968... 2024-09-13T22:45:58.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:57 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:57.912+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:58.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:57 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:57.912+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:58.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:57 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:57.912+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:58.097 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:57 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:57.912+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.523093+0000 front 2024-09-13T22:45:30.523137+0000 (oldest deadline 2024-09-13T22:45:56.423133+0000) 2024-09-13T22:45:58.597 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:58 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:58.125+0000 7f7e918ff640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T22:45:58.597 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:58 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:58.125+0000 7f7e918ff640 -1 osd.7 159 *** Got signal Terminated *** 2024-09-13T22:45:58.597 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:58 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:58.125+0000 7f7e918ff640 -1 osd.7 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T22:45:59.339 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:58 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:58.913+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:45:59.340 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:58 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:58.913+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:45:59.340 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:58 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:58.913+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:45:59.340 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:58 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:58.913+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.523093+0000 front 2024-09-13T22:45:30.523137+0000 (oldest deadline 2024-09-13T22:45:56.423133+0000) 2024-09-13T22:46:00.271 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:59 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:59.922+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:46:00.272 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:59 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:59.922+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:46:00.272 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:59 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:59.922+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:46:00.272 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:45:59 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:45:59.922+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.523093+0000 front 2024-09-13T22:45:30.523137+0000 (oldest deadline 2024-09-13T22:45:56.423133+0000) 2024-09-13T22:46:01.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:00 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:00.958+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:46:01.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:00 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:00.958+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:46:01.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:00 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:00.958+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:46:01.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:00 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:00.958+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.523093+0000 front 2024-09-13T22:45:30.523137+0000 (oldest deadline 2024-09-13T22:45:56.423133+0000) 2024-09-13T22:46:01.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:00 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:00.958+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.118:6804 osd.4 since back 2024-09-13T22:45:36.923949+0000 front 2024-09-13T22:45:36.924033+0000 (oldest deadline 2024-09-13T22:46:00.924309+0000) 2024-09-13T22:46:02.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:01 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:01.957+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:46:02.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:01 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:01.957+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:46:02.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:01 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:01.957+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:46:02.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:01 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:01.957+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.523093+0000 front 2024-09-13T22:45:30.523137+0000 (oldest deadline 2024-09-13T22:45:56.423133+0000) 2024-09-13T22:46:02.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:01 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:01.957+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.118:6804 osd.4 since back 2024-09-13T22:45:36.923949+0000 front 2024-09-13T22:45:36.924033+0000 (oldest deadline 2024-09-13T22:46:00.924309+0000) 2024-09-13T22:46:03.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:02 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:02.911+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6806 osd.0 since back 2024-09-13T22:45:03.520634+0000 front 2024-09-13T22:45:03.520677+0000 (oldest deadline 2024-09-13T22:45:28.220512+0000) 2024-09-13T22:46:03.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:02 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:02.911+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6814 osd.1 since back 2024-09-13T22:45:15.221979+0000 front 2024-09-13T22:45:15.222068+0000 (oldest deadline 2024-09-13T22:45:41.121942+0000) 2024-09-13T22:46:03.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:02 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:02.911+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6822 osd.2 since back 2024-09-13T22:45:21.122377+0000 front 2024-09-13T22:45:21.122415+0000 (oldest deadline 2024-09-13T22:45:47.022257+0000) 2024-09-13T22:46:03.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:02 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:02.911+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.105:6830 osd.3 since back 2024-09-13T22:45:30.523093+0000 front 2024-09-13T22:45:30.523137+0000 (oldest deadline 2024-09-13T22:45:56.423133+0000) 2024-09-13T22:46:03.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:02 smithi118 ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7[63205]: 2024-09-13T22:46:02.911+0000 7f7e89f0d640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.118:6804 osd.4 since back 2024-09-13T22:45:36.923949+0000 front 2024-09-13T22:45:36.924033+0000 (oldest deadline 2024-09-13T22:46:00.924309+0000) 2024-09-13T22:46:04.041 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:03 smithi118 podman[73888]: 2024-09-13 22:46:03.719510463 +0000 UTC m=+5.702091777 container died baaa3d182756ae1ee4b3cb87d2213cf31e977256167d6b35e0ca4de3044d35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-ecab7be, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T22:46:04.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:04 smithi118 podman[73888]: 2024-09-13 22:46:04.227152687 +0000 UTC m=+6.209734034 container cleanup baaa3d182756ae1ee4b3cb87d2213cf31e977256167d6b35e0ca4de3044d35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-13T22:46:04.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:04 smithi118 bash[73888]: ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7 2024-09-13T22:46:04.770 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:04 smithi118 podman[73947]: 2024-09-13 22:46:04.447284421 +0000 UTC m=+0.722354387 container remove baaa3d182756ae1ee4b3cb87d2213cf31e977256167d6b35e0ca4de3044d35a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be) 2024-09-13T22:46:05.089 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:04 smithi118 podman[74036]: 2024-09-13 22:46:04.671780416 +0000 UTC m=+0.015625993 image pull 0652ff2b5d78add567c2474cd2436ae21ad32722283bcbf25450e33edcd34e34 quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808 2024-09-13T22:46:05.089 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:04 smithi118 podman[74036]: 2024-09-13 22:46:04.77496115 +0000 UTC m=+0.118806728 container create 783cc4139d9588856d6e1796537c263c00bb1ff3496a1826a538334705945cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-ecab7be, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T22:46:05.089 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:04 smithi118 podman[74036]: 2024-09-13 22:46:04.988347205 +0000 UTC m=+0.332192793 container init 783cc4139d9588856d6e1796537c263c00bb1ff3496a1826a538334705945cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T22:46:05.089 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:04 smithi118 podman[74036]: 2024-09-13 22:46:04.991895842 +0000 UTC m=+0.335741427 container start 783cc4139d9588856d6e1796537c263c00bb1ff3496a1826a538334705945cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T22:46:05.089 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:05 smithi118 podman[74036]: 2024-09-13 22:46:05.08884862 +0000 UTC m=+0.432694198 container attach 783cc4139d9588856d6e1796537c263c00bb1ff3496a1826a538334705945cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=squid-ecab7be, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T22:46:05.347 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 13 22:46:05 smithi118 podman[74036]: 2024-09-13 22:46:05.179006132 +0000 UTC m=+0.522851714 container died 783cc4139d9588856d6e1796537c263c00bb1ff3496a1826a538334705945cfe (image=quay.ceph.io/ceph-ci/ceph@sha256:b893710f6933dd3672a2a633ce5ac92081563259207d21cd52290129179ca808, name=ceph-782bdfc8-721e-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-ecab7be, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T22:46:05.889 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-782bdfc8-721e-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T22:46:05.928 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T22:46:05.928 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T22:46:05.929 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T22:49:02.868 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T22:49:22.012 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:49:22.047 DEBUG:teuthology.orchestra.run.smithi118:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T22:49:22.075 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T22:49:22.077 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904293/remote/smithi105/crash 2024-09-13T22:49:22.078 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/crash -- . 2024-09-13T22:49:22.218 INFO:teuthology.orchestra.run.smithi105.stderr:tar: /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T22:49:22.218 INFO:teuthology.orchestra.run.smithi105.stderr:tar: Error is not recoverable: exiting now 2024-09-13T22:49:22.220 DEBUG:teuthology.misc:Transferring archived files from smithi118:/var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904293/remote/smithi118/crash 2024-09-13T22:49:22.221 DEBUG:teuthology.orchestra.run.smithi118:> sudo tar c -f - -C /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/crash -- . 2024-09-13T22:49:22.253 INFO:teuthology.orchestra.run.smithi118.stderr:tar: /var/lib/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T22:49:22.253 INFO:teuthology.orchestra.run.smithi118.stderr:tar: Error is not recoverable: exiting now 2024-09-13T22:49:22.254 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T22:49:22.255 DEBUG:teuthology.orchestra.run.smithi105:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-13T22:49:22.285 INFO:tasks.cephadm:Compressing logs... 2024-09-13T22:49:22.286 DEBUG:teuthology.orchestra.run.smithi105:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T22:49:22.328 DEBUG:teuthology.orchestra.run.smithi118:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T22:49:22.352 INFO:teuthology.orchestra.run.smithi105.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T22:49:22.352 INFO:teuthology.orchestra.run.smithi105.stderr:: No such file or directory 2024-09-13T22:49:22.352 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T22:49:22.358 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T22:49:22.358 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T22:49:22.358 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mon.smithi105.log 2024-09-13T22:49:22.358 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-volume.log: /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mgr.smithi105.swmmea.log 2024-09-13T22:49:22.358 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mon.smithi105.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T22:49:22.358 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mgr.smithi105.swmmea.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T22:49:22.358 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.log 2024-09-13T22:49:22.359 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.2.log: 85.8% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T22:49:22.359 INFO:teuthology.orchestra.run.smithi105.stderr: 91.5% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T22:49:22.361 INFO:teuthology.orchestra.run.smithi118.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T22:49:22.361 INFO:teuthology.orchestra.run.smithi118.stderr:: No such file or directory 2024-09-13T22:49:22.361 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi118.izbofj.log 2024-09-13T22:49:22.362 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T22:49:22.362 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T22:49:22.362 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi118.izbofj.log: /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T22:49:22.362 INFO:teuthology.orchestra.run.smithi118.stderr: 87.7% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi118.izbofj.log.gz 2024-09-13T22:49:22.362 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T22:49:22.362 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.4.log: /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mgr.smithi118.rbhtgz.log 2024-09-13T22:49:22.362 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T22:49:22.362 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mgr.smithi118.rbhtgz.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T22:49:22.363 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.6.log: 85.7% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.cephadm.log.gzgzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.log 2024-09-13T22:49:22.363 INFO:teuthology.orchestra.run.smithi118.stderr: 2024-09-13T22:49:22.363 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mon.smithi118.log 2024-09-13T22:49:22.363 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.5.log: /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.log: 91.6% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T22:49:22.363 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T22:49:22.363 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mon.smithi118.log: 87.0% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T22:49:22.367 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T22:49:22.368 INFO:teuthology.orchestra.run.smithi118.stderr: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T22:49:22.369 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi105.ddoxtb.log 2024-09-13T22:49:22.369 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T22:49:22.369 INFO:teuthology.orchestra.run.smithi105.stderr: 91.1% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T22:49:22.369 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi105.ddoxtb.log: 88.8% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi105.ddoxtb.log.gz 2024-09-13T22:49:22.369 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T22:49:22.372 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 87.0% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T22:49:22.405 INFO:teuthology.orchestra.run.smithi118.stderr: 89.9% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mgr.smithi118.rbhtgz.log.gz 2024-09-13T22:49:22.408 INFO:teuthology.orchestra.run.smithi105.stderr: 91.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T22:49:22.734 INFO:teuthology.orchestra.run.smithi105.stderr: 88.7% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mgr.smithi105.swmmea.log.gz 2024-09-13T22:49:22.834 INFO:teuthology.orchestra.run.smithi118.stderr: 92.6% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mon.smithi118.log.gz 2024-09-13T22:49:24.226 INFO:teuthology.orchestra.run.smithi105.stderr: 91.8% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-mon.smithi105.log.gz 2024-09-13T22:49:33.988 INFO:teuthology.orchestra.run.smithi105.stderr: 94.5% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T22:49:35.711 INFO:teuthology.orchestra.run.smithi118.stderr: 93.9% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T22:49:36.857 INFO:teuthology.orchestra.run.smithi118.stderr: 94.2% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T22:49:38.450 INFO:teuthology.orchestra.run.smithi105.stderr: 94.1% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T22:49:39.192 INFO:teuthology.orchestra.run.smithi105.stderr: 94.2% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T22:49:39.976 INFO:teuthology.orchestra.run.smithi105.stderr: 94.0% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T22:49:39.978 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-13T22:49:39.979 INFO:teuthology.orchestra.run.smithi105.stderr:real 0m17.637s 2024-09-13T22:49:39.979 INFO:teuthology.orchestra.run.smithi105.stderr:user 1m1.908s 2024-09-13T22:49:39.979 INFO:teuthology.orchestra.run.smithi105.stderr:sys 0m2.487s 2024-09-13T22:49:41.219 INFO:teuthology.orchestra.run.smithi118.stderr: 94.2% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T22:49:46.050 INFO:teuthology.orchestra.run.smithi118.stderr: 94.1% -- replaced with /var/log/ceph/782bdfc8-721e-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T22:49:46.051 INFO:teuthology.orchestra.run.smithi118.stderr: 2024-09-13T22:49:46.052 INFO:teuthology.orchestra.run.smithi118.stderr:real 0m23.708s 2024-09-13T22:49:46.052 INFO:teuthology.orchestra.run.smithi118.stderr:user 1m8.086s 2024-09-13T22:49:46.052 INFO:teuthology.orchestra.run.smithi118.stderr:sys 0m2.771s 2024-09-13T22:49:46.052 INFO:tasks.cephadm:Archiving logs... 2024-09-13T22:49:46.052 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904293/remote/smithi105/log 2024-09-13T22:49:46.053 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T22:49:54.696 DEBUG:teuthology.misc:Transferring archived files from smithi118:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904293/remote/smithi118/log 2024-09-13T22:49:54.697 DEBUG:teuthology.orchestra.run.smithi118:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T22:50:04.339 INFO:tasks.cephadm:Removing cluster... 2024-09-13T22:50:04.340 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 --force 2024-09-13T22:50:04.693 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 782bdfc8-721e-11ef-bceb-c7b262605968 --force 2024-09-13T22:50:05.066 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T22:50:05.066 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T22:50:05.082 DEBUG:teuthology.orchestra.run.smithi118:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T22:50:05.098 INFO:tasks.cephadm:Teardown complete 2024-09-13T22:50:05.098 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T22:50:05.107 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T22:50:05.107 DEBUG:teuthology.orchestra.run.smithi105:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:50:05.125 DEBUG:teuthology.orchestra.run.smithi118:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T22:50:05.137 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:50:05.142 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:50:05.142 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-13T22:50:05.142 INFO:teuthology.orchestra.run.smithi105.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 -123us[ -75us] +/- 62ms 2024-09-13T22:50:05.142 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 54 +140us[ +188us] +/- 59ms 2024-09-13T22:50:05.142 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 52 +8021ns[ +57us] +/- 60ms 2024-09-13T22:50:05.143 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:50:05.151 INFO:teuthology.orchestra.run.smithi118.stderr:bash: line 1: ntpq: command not found 2024-09-13T22:50:05.156 INFO:teuthology.orchestra.run.smithi118.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T22:50:05.156 INFO:teuthology.orchestra.run.smithi118.stdout:=============================================================================== 2024-09-13T22:50:05.156 INFO:teuthology.orchestra.run.smithi118.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 -120us[ -75us] +/- 62ms 2024-09-13T22:50:05.156 INFO:teuthology.orchestra.run.smithi118.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 53 +150us[ +194us] +/- 59ms 2024-09-13T22:50:05.156 INFO:teuthology.orchestra.run.smithi118.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 46 +15us[ +15us] +/- 60ms 2024-09-13T22:50:05.156 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T22:50:05.157 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T22:50:05.166 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T22:50:05.167 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T22:50:05.175 DEBUG:teuthology.orchestra.run.smithi105:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T22:50:05.261 DEBUG:teuthology.orchestra.run.smithi118:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T22:50:05.334 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T22:50:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:50:05.363 DEBUG:teuthology.orchestra.run.smithi118:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-13T22:50:05.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:50:05.392 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T22:50:05.403 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T22:50:05.412 INFO:teuthology.task.internal:Duration was 1963.520245 seconds 2024-09-13T22:50:05.413 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T22:50:05.425 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T22:50:05.426 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T22:50:05.429 DEBUG:teuthology.orchestra.run.smithi118:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T22:50:05.465 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:50:05.470 INFO:teuthology.orchestra.run.smithi118.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T22:50:05.884 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T22:50:05.885 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi105.front.sepia.ceph.com 2024-09-13T22:50:05.885 DEBUG:teuthology.orchestra.run.smithi105:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T22:50:05.927 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi118.front.sepia.ceph.com 2024-09-13T22:50:05.927 DEBUG:teuthology.orchestra.run.smithi118:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T22:50:05.954 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T22:50:05.954 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T22:50:05.970 DEBUG:teuthology.orchestra.run.smithi118:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T22:50:06.022 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T22:50:06.022 DEBUG:teuthology.orchestra.run.smithi105:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T22:50:06.038 DEBUG:teuthology.orchestra.run.smithi118:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T22:50:20.010 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T22:50:20.022 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T22:50:20.022 DEBUG:teuthology.orchestra.run.smithi105:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T22:50:20.047 DEBUG:teuthology.orchestra.run.smithi118:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T22:50:20.078 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T22:50:20.090 DEBUG:teuthology.orchestra.run.smithi105:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:50:20.093 DEBUG:teuthology.orchestra.run.smithi118:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:50:20.114 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = core 2024-09-13T22:50:20.148 INFO:teuthology.orchestra.run.smithi118.stdout:kernel.core_pattern = core 2024-09-13T22:50:20.181 DEBUG:teuthology.orchestra.run.smithi105:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:50:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:50:20.198 DEBUG:teuthology.orchestra.run.smithi118:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T22:50:20.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T22:50:20.240 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T22:50:20.251 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T22:50:20.252 DEBUG:teuthology.misc:Transferring archived files from smithi105:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904293/remote/smithi105 2024-09-13T22:50:20.252 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T22:50:20.315 DEBUG:teuthology.misc:Transferring archived files from smithi118:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-13_21:08:03-orch-squid-distro-default-smithi/7904293/remote/smithi118 2024-09-13T22:50:20.316 DEBUG:teuthology.orchestra.run.smithi118:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T22:50:20.360 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T22:50:20.361 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T22:50:20.363 DEBUG:teuthology.orchestra.run.smithi118:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T22:50:20.410 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T22:50:20.421 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T22:50:20.421 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T22:50:20.431 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T22:50:20.431 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T22:50:20.434 DEBUG:teuthology.orchestra.run.smithi118:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T22:50:20.447 INFO:teuthology.orchestra.run.smithi105.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 22:50 /home/ubuntu/cephtest 2024-09-13T22:50:20.467 INFO:teuthology.orchestra.run.smithi118.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 22:50 /home/ubuntu/cephtest 2024-09-13T22:50:20.468 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T22:50:20.479 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T22:50:20.508 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1963.5202445983887 owner: scheduled_teuthology@teuthology success: true 2024-09-13T22:50:20.509 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T22:50:20.598 INFO:teuthology.run:pass