2024-09-18T07:10:10.972 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-18T07:10:11.005 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-18T07:10:11.081 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi/7910284 branch: wip-guits-main-2024-09-17-1213 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7910284' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-main-2024-09-17-1213 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-guits-main-2024-09-17-1213 sha1: e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 8982 sha1: e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-main-2024-09-17-1213 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e targets: smithi083.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBA198IqbYe1eLgiXPJLGkofGVSuc/ipRfwCGNUhVzbRE7AR0Z157nvM546ZlVnX1fypfDIzISDLTxhV2hH1yjcg= smithi096.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPqgOWdL6XpIl2c8p7Nr84RFyG251JjZm8ZblxQLjS3sqLfZbL9L5ngtGS8Qew8ld5DmVOYCmK/4ZS1xIG2B1ME= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-18_06:59:44 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-18T07:10:11.081 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e/qa; will attempt to use it 2024-09-18T07:10:11.082 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e/qa/tasks 2024-09-18T07:10:11.082 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-18T07:10:11.084 INFO:teuthology.task.internal:Checking packages... 2024-09-18T07:10:11.105 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e' 2024-09-18T07:10:11.106 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-18T07:10:11.106 INFO:teuthology.packaging:ref: None 2024-09-18T07:10:11.106 INFO:teuthology.packaging:tag: None 2024-09-18T07:10:11.106 INFO:teuthology.packaging:branch: wip-guits-main-2024-09-17-1213 2024-09-18T07:10:11.106 INFO:teuthology.packaging:sha1: e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e 2024-09-18T07:10:11.106 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-main-2024-09-17-1213 2024-09-18T07:10:11.246 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4983.ge7fb7b56 2024-09-18T07:10:11.247 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-18T07:10:11.253 INFO:teuthology.task.internal:no buildpackages task found 2024-09-18T07:10:11.253 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-18T07:10:11.301 INFO:teuthology.task.internal:Saving configuration 2024-09-18T07:10:11.317 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-18T07:10:11.366 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-18T07:10:11.390 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi083.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi/7910284', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-18 07:05:12.841112', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBA198IqbYe1eLgiXPJLGkofGVSuc/ipRfwCGNUhVzbRE7AR0Z157nvM546ZlVnX1fypfDIzISDLTxhV2hH1yjcg='} 2024-09-18T07:10:11.413 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi096.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi/7910284', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-18 07:05:12.839069', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPqgOWdL6XpIl2c8p7Nr84RFyG251JjZm8ZblxQLjS3sqLfZbL9L5ngtGS8Qew8ld5DmVOYCmK/4ZS1xIG2B1ME='} 2024-09-18T07:10:11.413 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-18T07:10:11.418 INFO:teuthology.task.internal:roles: ubuntu@smithi083.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-18T07:10:11.419 INFO:teuthology.task.internal:roles: ubuntu@smithi096.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-18T07:10:11.419 INFO:teuthology.run_tasks:Running task console_log... 2024-09-18T07:10:11.526 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f559f50bbe0>, signals=[15]) 2024-09-18T07:10:11.526 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-18T07:10:11.532 INFO:teuthology.task.internal:Opening connections... 2024-09-18T07:10:11.532 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:10:11.534 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:10:11.611 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi096.front.sepia.ceph.com 2024-09-18T07:10:11.612 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:10:11.685 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-18T07:10:11.704 DEBUG:teuthology.orchestra.run.smithi083:> uname -m 2024-09-18T07:10:11.721 INFO:teuthology.orchestra.run.smithi083.stdout:x86_64 2024-09-18T07:10:11.722 DEBUG:teuthology.orchestra.run.smithi083:> cat /etc/os-release 2024-09-18T07:10:11.778 INFO:teuthology.orchestra.run.smithi083.stdout:NAME="CentOS Stream" 2024-09-18T07:10:11.778 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION="9" 2024-09-18T07:10:11.778 INFO:teuthology.orchestra.run.smithi083.stdout:ID="centos" 2024-09-18T07:10:11.778 INFO:teuthology.orchestra.run.smithi083.stdout:ID_LIKE="rhel fedora" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION_ID="9" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:PLATFORM_ID="platform:el9" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:ANSI_COLOR="0;31" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:LOGO="fedora-logo-icon" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:HOME_URL="https://centos.org/" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-18T07:10:11.779 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-18T07:10:11.780 INFO:teuthology.lock.ops:Updating smithi083.front.sepia.ceph.com on lock server 2024-09-18T07:10:11.816 DEBUG:teuthology.orchestra.run.smithi096:> uname -m 2024-09-18T07:10:11.834 INFO:teuthology.orchestra.run.smithi096.stdout:x86_64 2024-09-18T07:10:11.835 DEBUG:teuthology.orchestra.run.smithi096:> cat /etc/os-release 2024-09-18T07:10:11.893 INFO:teuthology.orchestra.run.smithi096.stdout:NAME="CentOS Stream" 2024-09-18T07:10:11.893 INFO:teuthology.orchestra.run.smithi096.stdout:VERSION="9" 2024-09-18T07:10:11.893 INFO:teuthology.orchestra.run.smithi096.stdout:ID="centos" 2024-09-18T07:10:11.893 INFO:teuthology.orchestra.run.smithi096.stdout:ID_LIKE="rhel fedora" 2024-09-18T07:10:11.893 INFO:teuthology.orchestra.run.smithi096.stdout:VERSION_ID="9" 2024-09-18T07:10:11.894 INFO:teuthology.orchestra.run.smithi096.stdout:PLATFORM_ID="platform:el9" 2024-09-18T07:10:11.894 INFO:teuthology.orchestra.run.smithi096.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-18T07:10:11.894 INFO:teuthology.orchestra.run.smithi096.stdout:ANSI_COLOR="0;31" 2024-09-18T07:10:11.894 INFO:teuthology.orchestra.run.smithi096.stdout:LOGO="fedora-logo-icon" 2024-09-18T07:10:11.894 INFO:teuthology.orchestra.run.smithi096.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-18T07:10:11.894 INFO:teuthology.orchestra.run.smithi096.stdout:HOME_URL="https://centos.org/" 2024-09-18T07:10:11.894 INFO:teuthology.orchestra.run.smithi096.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-18T07:10:11.894 INFO:teuthology.orchestra.run.smithi096.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-18T07:10:11.894 INFO:teuthology.orchestra.run.smithi096.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-18T07:10:11.894 INFO:teuthology.lock.ops:Updating smithi096.front.sepia.ceph.com on lock server 2024-09-18T07:10:11.924 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-18T07:10:11.936 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-18T07:10:11.964 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-18T07:10:11.964 DEBUG:teuthology.orchestra.run.smithi083:> test '!' -e /home/ubuntu/cephtest 2024-09-18T07:10:11.967 DEBUG:teuthology.orchestra.run.smithi096:> test '!' -e /home/ubuntu/cephtest 2024-09-18T07:10:11.984 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-18T07:10:12.009 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-18T07:10:12.009 DEBUG:teuthology.orchestra.run.smithi083:> test -z $(ls -A /var/lib/ceph) 2024-09-18T07:10:12.025 DEBUG:teuthology.orchestra.run.smithi096:> test -z $(ls -A /var/lib/ceph) 2024-09-18T07:10:12.190 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-18T07:10:12.260 INFO:teuthology.run_tasks:Running task kernel... 2024-09-18T07:10:12.295 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-18T07:10:12.295 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-18T07:10:12.296 DEBUG:teuthology.orchestra.run.smithi083:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-18T07:10:12.296 DEBUG:teuthology.orchestra.run.smithi096:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-18T07:10:12.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T07:10:12.314 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-09-18T07:10:12.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T07:10:12.315 DEBUG:teuthology.orchestra.run.smithi096:> uname -r 2024-09-18T07:10:12.373 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-503.el9.x86_64 2024-09-18T07:10:12.373 INFO:teuthology.task.kernel:Running kernel on smithi083: 5.14.0-503.el9.x86_64 2024-09-18T07:10:12.373 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum install -y kernel 2024-09-18T07:10:12.374 INFO:teuthology.orchestra.run.smithi096.stdout:5.14.0-503.el9.x86_64 2024-09-18T07:10:12.374 INFO:teuthology.task.kernel:Running kernel on smithi096: 5.14.0-503.el9.x86_64 2024-09-18T07:10:12.374 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum install -y kernel 2024-09-18T07:10:15.970 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - BaseOS 4.5 MB/s | 8.3 MB 00:01 2024-09-18T07:10:16.045 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - BaseOS 4.4 MB/s | 8.3 MB 00:01 2024-09-18T07:10:17.713 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - AppStream 35 MB/s | 20 MB 00:00 2024-09-18T07:10:17.738 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - AppStream 28 MB/s | 20 MB 00:00 2024-09-18T07:10:22.030 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-18T07:10:23.255 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - CRB 3.7 MB/s | 6.5 MB 00:01 2024-09-18T07:10:23.539 INFO:teuthology.orchestra.run.smithi096.stdout:CentOS Stream 9 - Extras packages 127 kB/s | 19 kB 00:00 2024-09-18T07:10:24.923 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - Extras packages 41 kB/s | 19 kB 00:00 2024-09-18T07:10:25.110 INFO:teuthology.orchestra.run.smithi096.stdout:Extra Packages for Enterprise Linux 16 MB/s | 23 MB 00:01 2024-09-18T07:10:26.210 INFO:teuthology.orchestra.run.smithi083.stdout:Extra Packages for Enterprise Linux 19 MB/s | 23 MB 00:01 2024-09-18T07:10:30.482 INFO:teuthology.orchestra.run.smithi096.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-18T07:10:31.633 INFO:teuthology.orchestra.run.smithi083.stdout:lab-extras 23 kB/s | 1.7 kB 00:00 2024-09-18T07:10:32.396 INFO:teuthology.orchestra.run.smithi096.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-18T07:10:32.396 INFO:teuthology.orchestra.run.smithi096.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-18T07:10:32.479 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2024-09-18T07:10:32.490 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-18T07:10:32.490 INFO:teuthology.orchestra.run.smithi096.stdout: Package Architecture Version Repository Size 2024-09-18T07:10:32.490 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-18T07:10:32.490 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2024-09-18T07:10:32.490 INFO:teuthology.orchestra.run.smithi096.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout:Install 4 Packages 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 79 M 2024-09-18T07:10:32.491 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 126 M 2024-09-18T07:10:32.492 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2024-09-18T07:10:32.862 INFO:teuthology.orchestra.run.smithi096.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 415 kB/s | 36 kB 00:00 2024-09-18T07:10:33.250 INFO:teuthology.orchestra.run.smithi096.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-18T07:10:33.523 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-18T07:10:33.523 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-18T07:10:33.632 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-09-18T07:10:33.639 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-18T07:10:33.640 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2024-09-18T07:10:33.640 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-18T07:10:33.640 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-09-18T07:10:33.640 INFO:teuthology.orchestra.run.smithi083.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-18T07:10:33.640 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-09-18T07:10:33.640 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-18T07:10:33.640 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-18T07:10:33.640 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-18T07:10:33.641 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:10:33.641 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-09-18T07:10:33.641 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-09-18T07:10:33.641 INFO:teuthology.orchestra.run.smithi083.stdout:Install 4 Packages 2024-09-18T07:10:33.641 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:10:33.642 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 79 M 2024-09-18T07:10:33.642 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 126 M 2024-09-18T07:10:33.642 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-09-18T07:10:33.716 INFO:teuthology.orchestra.run.smithi096.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 37 MB/s | 35 MB 00:00 2024-09-18T07:10:33.776 INFO:teuthology.orchestra.run.smithi083.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 327 kB/s | 36 kB 00:00 2024-09-18T07:10:34.042 INFO:teuthology.orchestra.run.smithi096.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-18T07:10:34.043 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2024-09-18T07:10:34.043 INFO:teuthology.orchestra.run.smithi096.stdout:Total 51 MB/s | 79 MB 00:01 2024-09-18T07:10:34.727 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2024-09-18T07:10:35.099 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2024-09-18T07:10:35.099 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2024-09-18T07:10:36.059 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2024-09-18T07:10:36.059 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2024-09-18T07:10:38.519 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2024-09-18T07:10:39.136 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-18T07:10:39.210 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T07:10:40.884 INFO:teuthology.orchestra.run.smithi083.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 2.2 MB/s | 16 MB 00:07 2024-09-18T07:10:41.975 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T07:10:42.103 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T07:10:46.133 INFO:teuthology.orchestra.run.smithi083.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 2.3 MB/s | 29 MB 00:12 2024-09-18T07:10:46.929 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T07:10:46.969 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:10:48.225 INFO:teuthology.orchestra.run.smithi083.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 2.4 MB/s | 35 MB 00:14 2024-09-18T07:10:48.226 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-09-18T07:10:48.226 INFO:teuthology.orchestra.run.smithi083.stdout:Total 5.4 MB/s | 79 MB 00:14 2024-09-18T07:10:48.911 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-09-18T07:10:49.247 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-09-18T07:10:49.248 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-09-18T07:10:50.199 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-09-18T07:10:50.199 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-09-18T07:10:51.818 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:10:52.611 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-09-18T07:10:53.245 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-18T07:10:53.337 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T07:10:56.079 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T07:10:56.214 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T07:11:01.060 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T07:11:01.190 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:06.059 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:30.351 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:30.390 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:30.678 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:30.678 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-18T07:11:30.679 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T07:11:30.679 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T07:11:31.494 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:31.494 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:11:31.494 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2024-09-18T07:11:31.494 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-18T07:11:31.494 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-18T07:11:31.494 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-18T07:11:31.494 INFO:teuthology.orchestra.run.smithi096.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-18T07:11:31.494 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:11:31.495 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2024-09-18T07:11:31.866 DEBUG:teuthology.orchestra.run.smithi096:> rpm -q kernel | sort -rV | head -n 1 2024-09-18T07:11:31.949 INFO:teuthology.orchestra.run.smithi096.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-18T07:11:31.949 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-18T07:11:31.949 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-18T07:11:31.950 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-18T07:11:31.950 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-18T07:11:31.950 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-18T07:11:31.950 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi096.front.sepia.ceph.com, path=None, version=distro) 2024-09-18T07:11:31.950 DEBUG:teuthology.orchestra.run.smithi096:> rpm -q kernel | sort -rV | head -n 1 2024-09-18T07:11:32.010 INFO:teuthology.orchestra.run.smithi096.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-18T07:11:32.170 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-18T07:11:32.170 DEBUG:teuthology.orchestra.run.smithi096:> sudo rpm -qi grub2-tools 2024-09-18T07:11:32.212 INFO:teuthology.orchestra.run.smithi096.stdout:Name : grub2-tools 2024-09-18T07:11:32.212 INFO:teuthology.orchestra.run.smithi096.stdout:Epoch : 1 2024-09-18T07:11:32.212 INFO:teuthology.orchestra.run.smithi096.stdout:Version : 2.06 2024-09-18T07:11:32.212 INFO:teuthology.orchestra.run.smithi096.stdout:Release : 82.el9 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Architecture: x86_64 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Group : Unspecified 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Size : 8274670 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:License : GPLv3+ 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Packager : builder@centos.org 2024-09-18T07:11:32.213 INFO:teuthology.orchestra.run.smithi096.stdout:Vendor : CentOS 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout:Summary : Support tools for GRUB. 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout:Description : 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout:hardware devices. 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:11:32.214 INFO:teuthology.orchestra.run.smithi096.stdout:This subpackage provides tools for support of all platforms. 2024-09-18T07:11:32.215 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-18T07:11:32.215 INFO:teuthology.task.kernel:Updating grub on smithi096 to boot 5.14.0-508.el9.x86_64 2024-09-18T07:11:32.216 DEBUG:teuthology.orchestra.run.smithi096:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-18T07:11:32.895 INFO:teuthology.orchestra.run.smithi096.stderr:Generating grub configuration file ... 2024-09-18T07:11:34.245 INFO:teuthology.orchestra.run.smithi096.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-18T07:11:34.287 INFO:teuthology.orchestra.run.smithi096.stderr:done 2024-09-18T07:11:34.289 DEBUG:teuthology.orchestra.run.smithi096:> mktemp 2024-09-18T07:11:34.305 INFO:teuthology.orchestra.run.smithi096.stdout:/tmp/tmp.b4BVgh1b8f 2024-09-18T07:11:34.305 DEBUG:teuthology.orchestra.run.smithi096:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.b4BVgh1b8f 2024-09-18T07:11:34.373 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod 0666 /tmp/tmp.b4BVgh1b8f 2024-09-18T07:11:34.514 DEBUG:teuthology.orchestra.remote:smithi096:/tmp/tmp.b4BVgh1b8f is 6KB 2024-09-18T07:11:34.563 DEBUG:teuthology.orchestra.run.smithi096:> rm -fr /tmp/tmp.b4BVgh1b8f 2024-09-18T07:11:34.578 DEBUG:teuthology.orchestra.run.smithi096:> sudo /bin/ls /boot/loader/entries || true 2024-09-18T07:11:34.660 INFO:teuthology.orchestra.run.smithi096.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-18T07:11:34.660 INFO:teuthology.orchestra.run.smithi096.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-18T07:11:34.660 INFO:teuthology.orchestra.run.smithi096.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-18T07:11:34.660 INFO:teuthology.orchestra.run.smithi096.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-18T07:11:34.661 DEBUG:teuthology.orchestra.run.smithi096:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-18T07:11:34.824 DEBUG:teuthology.orchestra.run.smithi096:> sudo shutdown -r now 2024-09-18T07:11:44.389 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:44.434 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:44.732 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:44.733 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-18T07:11:44.733 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T07:11:44.733 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T07:11:45.559 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T07:11:45.559 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:11:45.559 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-09-18T07:11:45.559 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-18T07:11:45.559 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-18T07:11:45.559 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-18T07:11:45.559 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-18T07:11:45.559 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:11:45.560 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-09-18T07:11:45.912 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-09-18T07:11:45.998 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-18T07:11:45.998 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-18T07:11:45.998 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-18T07:11:45.998 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-18T07:11:45.998 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-18T07:11:45.998 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-18T07:11:45.998 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi083.front.sepia.ceph.com, path=None, version=distro) 2024-09-18T07:11:45.998 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-09-18T07:11:46.057 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-18T07:11:46.058 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-18T07:11:46.058 DEBUG:teuthology.orchestra.run.smithi083:> sudo rpm -qi grub2-tools 2024-09-18T07:11:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:Name : grub2-tools 2024-09-18T07:11:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:Epoch : 1 2024-09-18T07:11:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:Version : 2.06 2024-09-18T07:11:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:Release : 82.el9 2024-09-18T07:11:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:Architecture: x86_64 2024-09-18T07:11:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-18T07:11:46.090 INFO:teuthology.orchestra.run.smithi083.stdout:Group : Unspecified 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:Size : 8274670 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:License : GPLv3+ 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:Packager : builder@centos.org 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:Vendor : CentOS 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:Summary : Support tools for GRUB. 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:Description : 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:11:46.091 INFO:teuthology.orchestra.run.smithi083.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-18T07:11:46.092 INFO:teuthology.orchestra.run.smithi083.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-18T07:11:46.092 INFO:teuthology.orchestra.run.smithi083.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-18T07:11:46.092 INFO:teuthology.orchestra.run.smithi083.stdout:hardware devices. 2024-09-18T07:11:46.092 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:11:46.092 INFO:teuthology.orchestra.run.smithi083.stdout:This subpackage provides tools for support of all platforms. 2024-09-18T07:11:46.093 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-18T07:11:46.093 INFO:teuthology.task.kernel:Updating grub on smithi083 to boot 5.14.0-508.el9.x86_64 2024-09-18T07:11:46.093 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-18T07:11:46.711 INFO:teuthology.orchestra.run.smithi083.stderr:Generating grub configuration file ... 2024-09-18T07:11:48.051 INFO:teuthology.orchestra.run.smithi083.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-18T07:11:48.086 INFO:teuthology.orchestra.run.smithi083.stderr:done 2024-09-18T07:11:48.088 DEBUG:teuthology.orchestra.run.smithi083:> mktemp 2024-09-18T07:11:48.104 INFO:teuthology.orchestra.run.smithi083.stdout:/tmp/tmp.h2MlGHs6zd 2024-09-18T07:11:48.104 DEBUG:teuthology.orchestra.run.smithi083:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.h2MlGHs6zd 2024-09-18T07:11:48.171 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0666 /tmp/tmp.h2MlGHs6zd 2024-09-18T07:11:48.314 DEBUG:teuthology.orchestra.remote:smithi083:/tmp/tmp.h2MlGHs6zd is 6KB 2024-09-18T07:11:48.364 DEBUG:teuthology.orchestra.run.smithi083:> rm -fr /tmp/tmp.h2MlGHs6zd 2024-09-18T07:11:48.379 DEBUG:teuthology.orchestra.run.smithi083:> sudo /bin/ls /boot/loader/entries || true 2024-09-18T07:11:48.441 INFO:teuthology.orchestra.run.smithi083.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-18T07:11:48.442 INFO:teuthology.orchestra.run.smithi083.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-18T07:11:48.442 INFO:teuthology.orchestra.run.smithi083.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-18T07:11:48.442 INFO:teuthology.orchestra.run.smithi083.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-18T07:11:48.443 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-18T07:11:48.608 DEBUG:teuthology.orchestra.run.smithi083:> sudo shutdown -r now 2024-09-18T07:12:04.841 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-18T07:12:04.842 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-18T07:12:04.843 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:12:18.622 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-18T07:12:18.623 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-18T07:12:18.624 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:12:23.357 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.96 2024-09-18T07:12:32.366 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-18T07:12:32.368 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:12:37.185 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.83 2024-09-18T07:12:46.193 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-18T07:12:46.193 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:12:49.249 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.83 2024-09-18T07:13:01.261 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-18T07:13:01.262 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:13:19.677 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.83 2024-09-18T07:13:32.381 DEBUG:teuthology.orchestra.remote:timed out 2024-09-18T07:13:34.680 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-18T07:13:34.682 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:13:35.053 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-09-18T07:13:35.475 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-18T07:13:35.475 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-18T07:13:35.476 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-09-18T07:13:35.491 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-508.el9.x86_64 2024-09-18T07:13:35.491 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-18T07:13:35.491 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-18T07:13:35.491 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-18T07:13:36.492 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-18T07:13:36.492 DEBUG:teuthology.orchestra.run.smithi083:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-18T07:13:36.563 INFO:teuthology.orchestra.run.smithi083.stdout:ttyS1 2024-09-18T07:13:36.583 DEBUG:teuthology.parallel:result is None 2024-09-18T07:13:44.382 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-18T07:13:44.383 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:13:44.743 DEBUG:teuthology.orchestra.run.smithi096:> true 2024-09-18T07:13:45.085 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-18T07:13:45.181 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-18T07:13:45.181 DEBUG:teuthology.orchestra.run.smithi096:> uname -r 2024-09-18T07:13:45.198 INFO:teuthology.orchestra.run.smithi096.stdout:5.14.0-508.el9.x86_64 2024-09-18T07:13:45.198 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-18T07:13:45.198 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-18T07:13:45.198 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-18T07:13:46.199 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-18T07:13:46.199 DEBUG:teuthology.orchestra.run.smithi096:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-18T07:13:46.560 INFO:teuthology.orchestra.run.smithi096.stdout:ttyS1 2024-09-18T07:13:46.581 DEBUG:teuthology.parallel:result is None 2024-09-18T07:13:46.581 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-18T07:13:46.592 INFO:teuthology.task.internal:Creating test directory... 2024-09-18T07:13:46.592 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-18T07:13:46.595 DEBUG:teuthology.orchestra.run.smithi096:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-18T07:13:46.613 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-18T07:13:46.620 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-18T07:13:46.628 INFO:teuthology.task.internal:Creating archive directory... 2024-09-18T07:13:46.628 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-18T07:13:46.655 DEBUG:teuthology.orchestra.run.smithi096:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-18T07:13:46.697 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-18T07:13:46.703 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-18T07:13:46.704 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-18T07:13:46.736 DEBUG:teuthology.orchestra.run.smithi096:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-18T07:13:46.767 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-18T07:13:46.772 INFO:teuthology.orchestra.run.smithi096.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-18T07:13:46.778 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-18T07:13:46.784 INFO:teuthology.orchestra.run.smithi096.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-18T07:13:46.786 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-18T07:13:46.793 INFO:teuthology.task.internal:Configuring sudo... 2024-09-18T07:13:46.793 DEBUG:teuthology.orchestra.run.smithi083:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-18T07:13:46.823 DEBUG:teuthology.orchestra.run.smithi096:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-18T07:13:46.855 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-18T07:13:46.865 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-18T07:13:46.865 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-18T07:13:46.889 DEBUG:teuthology.orchestra.run.smithi096:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-18T07:13:46.913 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-18T07:13:46.988 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-18T07:13:47.034 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-18T07:13:47.116 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-18T07:13:47.158 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:13:47.158 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-18T07:13:47.223 DEBUG:teuthology.orchestra.run.smithi096:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-18T07:13:47.269 DEBUG:teuthology.orchestra.run.smithi096:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-18T07:13:47.318 DEBUG:teuthology.orchestra.run.smithi096:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-18T07:13:47.403 DEBUG:teuthology.orchestra.run.smithi096:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-18T07:13:47.446 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-18T07:13:47.446 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-18T07:13:47.513 DEBUG:teuthology.orchestra.run.smithi083:> sudo service rsyslog restart 2024-09-18T07:13:47.516 DEBUG:teuthology.orchestra.run.smithi096:> sudo service rsyslog restart 2024-09-18T07:13:47.564 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-18T07:13:47.607 INFO:teuthology.orchestra.run.smithi096.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-18T07:13:48.069 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-18T07:13:48.077 INFO:teuthology.task.internal:Starting timer... 2024-09-18T07:13:48.077 INFO:teuthology.run_tasks:Running task pcp... 2024-09-18T07:13:48.088 INFO:teuthology.run_tasks:Running task selinux... 2024-09-18T07:13:48.096 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-18T07:13:48.097 DEBUG:teuthology.orchestra.run.smithi083:> sudo service auditd rotate 2024-09-18T07:13:48.139 INFO:teuthology.orchestra.run.smithi083.stdout:Rotating logs: 2024-09-18T07:13:48.141 DEBUG:teuthology.orchestra.run.smithi096:> sudo service auditd rotate 2024-09-18T07:13:48.199 INFO:teuthology.orchestra.run.smithi096.stdout:Rotating logs: 2024-09-18T07:13:48.201 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-18T07:13:48.201 DEBUG:teuthology.orchestra.run.smithi083:> /usr/sbin/getenforce 2024-09-18T07:13:48.237 INFO:teuthology.orchestra.run.smithi083.stdout:Permissive 2024-09-18T07:13:48.238 DEBUG:teuthology.orchestra.run.smithi096:> /usr/sbin/getenforce 2024-09-18T07:13:48.261 INFO:teuthology.orchestra.run.smithi096.stdout:Permissive 2024-09-18T07:13:48.261 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi083.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi096.front.sepia.ceph.com': 'permissive'} 2024-09-18T07:13:48.261 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-18T07:13:48.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T07:13:48.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-18T07:13:48.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T07:13:48.333 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-18T07:13:48.333 DEBUG:teuthology.orchestra.run.smithi083:> sudo /usr/sbin/setenforce permissive 2024-09-18T07:13:48.379 DEBUG:teuthology.orchestra.run.smithi096:> sudo /usr/sbin/setenforce permissive 2024-09-18T07:13:48.416 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-18T07:13:48.426 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-18T07:13:48.429 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-18T07:13:48.547 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-18T07:13:48.559 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-18T07:13:48.560 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi083.front.sepia.ceph.com,smithi096.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-18T07:17:32.108 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi083.front.sepia.ceph.com'), Remote(name='ubuntu@smithi096.front.sepia.ceph.com')] 2024-09-18T07:17:32.110 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-18T07:17:32.111 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:17:32.189 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-09-18T07:17:32.264 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-09-18T07:17:32.264 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-18T07:17:32.265 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T07:17:32.338 DEBUG:teuthology.orchestra.run.smithi096:> true 2024-09-18T07:17:32.374 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi096.front.sepia.ceph.com' 2024-09-18T07:17:32.374 INFO:teuthology.run_tasks:Running task clock... 2024-09-18T07:17:32.385 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-18T07:17:32.386 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-18T07:17:32.386 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T07:17:32.389 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-18T07:17:32.389 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T07:17:32.416 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-18T07:17:32.430 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-18T07:17:32.445 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-18T07:17:32.454 INFO:teuthology.orchestra.run.smithi083.stderr:sudo: ntpd: command not found 2024-09-18T07:17:32.459 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-18T07:17:32.465 INFO:teuthology.orchestra.run.smithi083.stdout:506 Cannot talk to daemon 2024-09-18T07:17:32.479 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-18T07:17:32.483 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: ntpd: command not found 2024-09-18T07:17:32.492 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-18T07:17:32.494 INFO:teuthology.orchestra.run.smithi096.stdout:506 Cannot talk to daemon 2024-09-18T07:17:32.509 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-18T07:17:32.523 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-18T07:17:32.550 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-09-18T07:17:32.553 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-18T07:17:32.553 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-09-18T07:17:32.553 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:17:32.553 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:17:32.553 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:17:32.553 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:17:32.585 INFO:teuthology.orchestra.run.smithi096.stderr:bash: line 1: ntpq: command not found 2024-09-18T07:17:32.588 INFO:teuthology.orchestra.run.smithi096.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-18T07:17:32.588 INFO:teuthology.orchestra.run.smithi096.stdout:=============================================================================== 2024-09-18T07:17:32.588 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:17:32.588 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:17:32.588 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:17:32.588 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:17:32.589 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-18T07:17:32.685 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': 'e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e'} 2024-09-18T07:17:32.685 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-18T07:17:32.686 INFO:tasks.cephadm:Cluster fsid is 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:17:32.686 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-18T07:17:32.686 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-18T07:17:32.686 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi083': '172.21.15.83', 'mon.smithi096': '172.21.15.96'} 2024-09-18T07:17:32.687 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-18T07:17:32.687 DEBUG:teuthology.orchestra.run.smithi083:> sudo hostname $(hostname -s) 2024-09-18T07:17:32.712 DEBUG:teuthology.orchestra.run.smithi096:> sudo hostname $(hostname -s) 2024-09-18T07:17:32.737 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-18T07:17:32.738 DEBUG:teuthology.orchestra.run.smithi083:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-18T07:17:33.012 INFO:teuthology.orchestra.run.smithi083.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 18 07:17 /home/ubuntu/cephtest/cephadm 2024-09-18T07:17:33.012 DEBUG:teuthology.orchestra.run.smithi096:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-18T07:17:33.291 INFO:teuthology.orchestra.run.smithi096.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 18 07:17 /home/ubuntu/cephtest/cephadm 2024-09-18T07:17:33.291 DEBUG:teuthology.orchestra.run.smithi083:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-18T07:17:33.308 DEBUG:teuthology.orchestra.run.smithi096:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-18T07:17:33.341 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-18T07:17:33.341 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-18T07:17:33.350 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-18T07:17:34.922 INFO:teuthology.orchestra.run.smithi096.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-18T07:17:34.976 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-18T07:17:46.147 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:17:46.148 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-18T07:17:46.148 INFO:teuthology.orchestra.run.smithi083.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-18T07:17:46.148 INFO:teuthology.orchestra.run.smithi083.stdout: "repo_digests": [ 2024-09-18T07:17:46.148 INFO:teuthology.orchestra.run.smithi083.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-18T07:17:46.148 INFO:teuthology.orchestra.run.smithi083.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-18T07:17:46.148 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-18T07:17:46.148 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:17:46.257 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2024-09-18T07:17:46.257 INFO:teuthology.orchestra.run.smithi096.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-18T07:17:46.257 INFO:teuthology.orchestra.run.smithi096.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-18T07:17:46.257 INFO:teuthology.orchestra.run.smithi096.stdout: "repo_digests": [ 2024-09-18T07:17:46.258 INFO:teuthology.orchestra.run.smithi096.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-18T07:17:46.258 INFO:teuthology.orchestra.run.smithi096.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-18T07:17:46.258 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2024-09-18T07:17:46.258 INFO:teuthology.orchestra.run.smithi096.stdout:} 2024-09-18T07:17:46.276 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /etc/ceph 2024-09-18T07:17:46.303 DEBUG:teuthology.orchestra.run.smithi096:> sudo mkdir -p /etc/ceph 2024-09-18T07:17:46.330 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 777 /etc/ceph 2024-09-18T07:17:46.369 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod 777 /etc/ceph 2024-09-18T07:17:46.394 INFO:tasks.cephadm:Writing seed config... 2024-09-18T07:17:46.395 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-18T07:17:46.396 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-18T07:17:46.396 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-18T07:17:46.396 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-18T07:17:46.396 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-18T07:17:46.396 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-18T07:17:46.396 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-18T07:17:46.396 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-18T07:17:46.397 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:17:46.397 DEBUG:teuthology.orchestra.run.smithi083:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-18T07:17:46.425 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 126f471a-758e-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-18T07:17:46.425 DEBUG:teuthology.orchestra.run.smithi083:mon.smithi083> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi083.service 2024-09-18T07:17:46.468 INFO:tasks.cephadm:Bootstrapping... 2024-09-18T07:17:46.468 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 126f471a-758e-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.83 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-18T07:17:46.656 INFO:teuthology.orchestra.run.smithi083.stderr:-------------------------------------------------------------------------------- 2024-09-18T07:17:46.657 INFO:teuthology.orchestra.run.smithi083.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '126f471a-758e-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.83', '--skip-admin-label'] 2024-09-18T07:17:46.678 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 5.2.2 2024-09-18T07:17:46.680 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying podman|docker is present... 2024-09-18T07:17:46.700 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 5.2.2 2024-09-18T07:17:46.702 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying lvm2 is present... 2024-09-18T07:17:46.702 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying time synchronization is in place... 2024-09-18T07:17:46.709 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-18T07:17:46.715 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-09-18T07:17:46.723 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2024-09-18T07:17:46.729 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2024-09-18T07:17:46.729 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2024-09-18T07:17:46.729 INFO:teuthology.orchestra.run.smithi083.stderr:Repeating the final host check... 2024-09-18T07:17:46.749 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 5.2.2 2024-09-18T07:17:46.753 INFO:teuthology.orchestra.run.smithi083.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-18T07:17:46.753 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl is present 2024-09-18T07:17:46.753 INFO:teuthology.orchestra.run.smithi083.stderr:lvcreate is present 2024-09-18T07:17:46.757 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-18T07:17:46.763 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-09-18T07:17:46.770 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2024-09-18T07:17:46.776 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2024-09-18T07:17:46.777 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2024-09-18T07:17:46.777 INFO:teuthology.orchestra.run.smithi083.stderr:Host looks OK 2024-09-18T07:17:46.777 INFO:teuthology.orchestra.run.smithi083.stderr:Cluster fsid: 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:17:46.777 INFO:teuthology.orchestra.run.smithi083.stderr:Acquiring lock 140701767710608 on /run/cephadm/126f471a-758e-11ef-bceb-c7b262605968.lock 2024-09-18T07:17:46.777 INFO:teuthology.orchestra.run.smithi083.stderr:Lock 140701767710608 acquired on /run/cephadm/126f471a-758e-11ef-bceb-c7b262605968.lock 2024-09-18T07:17:46.778 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 3300 ... 2024-09-18T07:17:46.778 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 6789 ... 2024-09-18T07:17:46.778 INFO:teuthology.orchestra.run.smithi083.stderr:Base mon IP is 172.21.15.83, final addrv is [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2024-09-18T07:17:46.780 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.83 metric 100 2024-09-18T07:17:46.780 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.83 metric 100 2024-09-18T07:17:46.783 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-18T07:17:46.783 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-18T07:17:46.783 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-18T07:17:46.785 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-18T07:17:46.785 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-18T07:17:46.786 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-18T07:17:46.786 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-18T07:17:46.786 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:ce91/64 scope link noprefixroute 2024-09-18T07:17:46.786 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-18T07:17:46.786 INFO:teuthology.orchestra.run.smithi083.stderr:Mon IP `172.21.15.83` is in CIDR network `172.21.0.0/20` 2024-09-18T07:17:46.786 INFO:teuthology.orchestra.run.smithi083.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-18T07:17:46.787 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-18T07:17:46.885 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-18T07:17:47.273 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Getting image source signatures 2024-09-18T07:17:47.273 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-18T07:17:47.273 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-18T07:17:47.273 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-18T07:17:47.273 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-18T07:17:47.273 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-18T07:17:47.273 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-18T07:17:47.426 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Writing manifest to image destination 2024-09-18T07:17:47.571 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-18T07:17:47.950 INFO:teuthology.orchestra.run.smithi083.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-18T07:17:48.486 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-18T07:17:48.486 INFO:teuthology.orchestra.run.smithi083.stderr:Extracting ceph user uid/gid from container image... 2024-09-18T07:17:48.863 INFO:teuthology.orchestra.run.smithi083.stderr:stat: 167 167 2024-09-18T07:17:49.459 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial keys... 2024-09-18T07:17:49.816 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQCdfupmIKqDLBAAloODID29Y5ZrtwOQT/u7MQ== 2024-09-18T07:17:50.722 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQCefupmUhvrJRAAirbF86ebFCG1285Cot9RBg== 2024-09-18T07:17:51.611 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQCffupmYwtNIBAASBn9lOw+l5vXV7J34hISVA== 2024-09-18T07:17:52.127 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial monmap... 2024-09-18T07:17:52.542 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-18T07:17:52.542 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-18T07:17:52.542 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:17:52.542 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-18T07:17:52.981 INFO:teuthology.orchestra.run.smithi083.stderr:monmaptool for smithi083 [v2:172.21.15.83:3300,v1:172.21.15.83:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-18T07:17:52.981 INFO:teuthology.orchestra.run.smithi083.stderr:setting min_mon_release = octopus 2024-09-18T07:17:52.981 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: set fsid to 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:17:52.981 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-18T07:17:52.981 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:17:52.982 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mon... 2024-09-18T07:17:53.930 INFO:teuthology.orchestra.run.smithi083.stderr:create mon.smithi083 on 2024-09-18T07:17:54.169 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-18T07:17:54.325 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-126f471a-758e-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-126f471a-758e-11ef-bceb-c7b262605968.target. 2024-09-18T07:17:54.325 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-126f471a-758e-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-126f471a-758e-11ef-bceb-c7b262605968.target. 2024-09-18T07:17:54.649 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi083.service: Unit ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi083.service not loaded. 2024-09-18T07:17:54.657 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-126f471a-758e-11ef-bceb-c7b262605968.target.wants/ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi083.service → /etc/systemd/system/ceph-126f471a-758e-11ef-bceb-c7b262605968@.service. 2024-09-18T07:17:55.491 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-09-18T07:17:55.498 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-09-18T07:17:55.498 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-09-18T07:17:55.498 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2024-09-18T07:17:55.499 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon to start... 2024-09-18T07:17:55.499 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon... 2024-09-18T07:17:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:17:58 smithi083 ceph-mon[23371]: from='client.? 172.21.15.83:0/3948208513' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-18T07:17:59.417 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: cluster: 2024-09-18T07:17:59.417 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: id: 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: services: 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi083 (age 3s) 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: data: 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pgs: 2024-09-18T07:17:59.418 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:05.170 INFO:teuthology.orchestra.run.smithi083.stderr:mon is available 2024-09-18T07:18:05.170 INFO:teuthology.orchestra.run.smithi083.stderr:Assimilating anything we can from ceph.conf... 2024-09-18T07:18:07.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:07.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2024-09-18T07:18:07.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:18:07.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2024-09-18T07:18:07.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-18T07:18:07.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-18T07:18:07.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-18T07:18:07.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-18T07:18:07.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:07.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2024-09-18T07:18:07.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-18T07:18:07.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:07.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2024-09-18T07:18:07.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-18T07:18:07.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-18T07:18:07.667 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:07 smithi083 ceph-mon[23371]: from='client.? 172.21.15.83:0/3848843804' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-18T07:18:07.667 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:07 smithi083 ceph-mon[23371]: from='client.? 172.21.15.83:0/3848843804' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-18T07:18:07.903 INFO:teuthology.orchestra.run.smithi083.stderr:Generating new minimal ceph.conf... 2024-09-18T07:18:08.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:08 smithi083 ceph-mon[23371]: from='client.? 172.21.15.83:0/4130991205' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:18:08.936 INFO:teuthology.orchestra.run.smithi083.stderr:Restarting the monitor... 2024-09-18T07:18:09.312 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:08 smithi083 systemd[1]: Stopping Ceph mon.smithi083 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:18:09.312 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:08 smithi083 bash[23591]: Error: no container with name or ID "ceph-126f471a-758e-11ef-bceb-c7b262605968-mon.smithi083" found: no such container 2024-09-18T07:18:09.312 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083[23367]: 2024-09-18T07:18:09.084+0000 7fef602ca700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi083 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:18:09.312 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083[23367]: 2024-09-18T07:18:09.084+0000 7fef602ca700 -1 mon.smithi083@0(leader) e1 *** Got Signal Terminated *** 2024-09-18T07:18:09.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 podman[23603]: 2024-09-18 07:18:09.309433162 +0000 UTC m=+0.324943455 container died 88db8e82fdfd7895f9b6fd7dbe978bd26713c73683006013e485e6c379281802 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vendor=Red Hat, Inc., release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, RELEASE=HEAD, ceph=True, architecture=x86_64, io.buildah.version=1.19.8) 2024-09-18T07:18:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 podman[23603]: 2024-09-18 07:18:09.658622752 +0000 UTC m=+0.674133032 container cleanup 88db8e82fdfd7895f9b6fd7dbe978bd26713c73683006013e485e6c379281802 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vendor=Red Hat, Inc., architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, release=754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, ceph=True, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-type=git, GIT_CLEAN=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-18T07:18:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 bash[23603]: ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083 2024-09-18T07:18:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 bash[23629]: Error: no container with name or ID "ceph-126f471a-758e-11ef-bceb-c7b262605968-mon.smithi083" found: no such container 2024-09-18T07:18:10.250 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 podman[23617]: 2024-09-18 07:18:09.951423531 +0000 UTC m=+0.635438647 container remove 88db8e82fdfd7895f9b6fd7dbe978bd26713c73683006013e485e6c379281802 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, distribution-scope=public, GIT_CLEAN=True, name=centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, ceph=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , version=8, CEPH_POINT_RELEASE=-17.2.0, release=754, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-18T07:18:10.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi083.service: Deactivated successfully. 2024-09-18T07:18:10.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 systemd[1]: Stopped Ceph mon.smithi083 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:18:10.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:09 smithi083 systemd[1]: Starting Ceph mon.smithi083 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:18:10.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 podman[23692]: 2024-09-18 07:18:10.160643445 +0000 UTC m=+0.094186944 container create 9f77aab8b41c29566ed04a6ad5d13afde2a31c1b8335c525b209784b131bddab (image=quay.io/ceph/ceph:v17.2.0, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, architecture=x86_64, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0) 2024-09-18T07:18:10.251 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 podman[23692]: 2024-09-18 07:18:10.082800379 +0000 UTC m=+0.016343851 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-18T07:18:10.435 INFO:teuthology.orchestra.run.smithi083.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-18T07:18:10.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 podman[23692]: 2024-09-18 07:18:10.346224786 +0000 UTC m=+0.279768247 container init 9f77aab8b41c29566ed04a6ad5d13afde2a31c1b8335c525b209784b131bddab (image=quay.io/ceph/ceph:v17.2.0, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, release=754, version=8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, io.buildah.version=1.19.8, GIT_CLEAN=True, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True) 2024-09-18T07:18:10.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 podman[23692]: 2024-09-18 07:18:10.349275836 +0000 UTC m=+0.282819292 container start 9f77aab8b41c29566ed04a6ad5d13afde2a31c1b8335c525b209784b131bddab (image=quay.io/ceph/ceph:v17.2.0, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , release=754, ceph=True, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD) 2024-09-18T07:18:10.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: set uid:gid to 167:167 (ceph:ceph) 2024-09-18T07:18:10.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-18T07:18:10.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: pidfile_write: ignore empty --pid-file 2024-09-18T07:18:10.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: load: jerasure load: lrc 2024-09-18T07:18:10.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: RocksDB version: 6.15.5 2024-09-18T07:18:10.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-18T07:18:10.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Compile date Apr 18 2022 2024-09-18T07:18:10.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: DB SUMMARY 2024-09-18T07:18:10.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: DB Session ID: COH9DDNB41WI1M9Z196I 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: CURRENT file: CURRENT 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: IDENTITY file: IDENTITY 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi083/store.db dir, Total Num: 1, files: 000008.sst 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi083/store.db: 000010.log size: 84012 ; 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.error_if_exists: 0 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.create_if_missing: 0 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.paranoid_checks: 1 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.env: 0x55b9c36c6860 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.fs: Posix File System 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.info_log: 0x55b9c4093fa0 2024-09-18T07:18:10.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_file_opening_threads: 16 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.statistics: (nil) 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.use_fsync: 0 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_log_file_size: 0 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.keep_log_file_num: 1000 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.recycle_log_file_num: 0 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.allow_fallocate: 1 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.allow_mmap_reads: 0 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.allow_mmap_writes: 0 2024-09-18T07:18:10.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.use_direct_reads: 0 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.create_missing_column_families: 0 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.db_log_dir: 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi083/store.db 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.advise_random_on_open: 1 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.db_write_buffer_size: 0 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.write_buffer_manager: 0x55b9c4184330 2024-09-18T07:18:10.622 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.rate_limiter: (nil) 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.wal_recovery_mode: 2 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.enable_thread_tracking: 0 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.enable_pipelined_write: 0 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.unordered_write: 0 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-18T07:18:10.623 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.row_cache: None 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.wal_filter: None 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.allow_ingest_behind: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.preserve_deletes: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.two_write_queues: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.manual_wal_flush: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.atomic_flush: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.log_readahead_size: 0 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-18T07:18:10.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.best_efforts_recovery: 0 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.allow_data_in_errors: 0 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.db_host_id: __hostname__ 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_background_jobs: 2 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_background_compactions: -1 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_subcompactions: 1 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-18T07:18:10.625 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-18T07:18:10.626 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_total_wal_size: 0 2024-09-18T07:18:10.626 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_open_files: -1 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bytes_per_sync: 0 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_readahead_size: 0 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_background_flushes: -1 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Compression algorithms supported: 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: kZSTD supported: 0 2024-09-18T07:18:10.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: kXpressCompression supported: 0 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: kLZ4HCCompression supported: 1 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: kLZ4Compression supported: 1 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: kBZip2Compression supported: 0 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: kZlibCompression supported: 1 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: kSnappyCompression supported: 1 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000009 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.merge_operator: 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_filter: None 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_filter_factory: None 2024-09-18T07:18:10.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.sst_partitioner_factory: None 2024-09-18T07:18:10.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-18T07:18:10.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-18T07:18:10.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b9c405fd00) 2024-09-18T07:18:10.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks: 1 2024-09-18T07:18:10.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-18T07:18:10.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-18T07:18:10.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_top_level_index_and_filter: 1 2024-09-18T07:18:10.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_type: 0 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_index_type: 0 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_shortening: 1 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: hash_index_allow_collision: 1 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: checksum: 1 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: no_block_cache: 0 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache: 0x55b9c40ca170 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_name: BinnedLRUCache 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_options: 2024-09-18T07:18:10.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: capacity : 536870912 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: num_shard_bits : 4 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: strict_capacity_limit : 0 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: high_pri_pool_ratio: 0.000 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_compressed: (nil) 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: persistent_cache: (nil) 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size: 4096 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size_deviation: 10 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_restart_interval: 16 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_block_restart_interval: 1 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: metadata_block_size: 4096 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: partition_filters: 0 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: use_delta_encoding: 1 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-18T07:18:10.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: whole_key_filtering: 1 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: verify_compression: 0 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: read_amp_bytes_per_bit: 0 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: format_version: 4 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_index_compression: 1 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_align: 0 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.write_buffer_size: 33554432 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_write_buffer_number: 2 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compression: NoCompression 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bottommost_compression: Disabled 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.prefix_extractor: nullptr 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.num_levels: 7 2024-09-18T07:18:10.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-18T07:18:10.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compression_opts.level: 32767 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compression_opts.strategy: 0 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compression_opts.enabled: false 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.target_file_size_base: 67108864 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-18T07:18:10.634 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-18T07:18:10.635 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-18T07:18:10.635 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-18T07:18:10.635 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-18T07:18:10.635 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-18T07:18:10.635 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-18T07:18:10.635 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-18T07:18:10.635 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-18T07:18:10.635 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-18T07:18:10.635 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.arena_block_size: 4194304 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.disable_auto_compactions: 0 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-18T07:18:10.636 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-18T07:18:10.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-18T07:18:10.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-18T07:18:10.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-18T07:18:10.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-18T07:18:10.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-18T07:18:10.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-18T07:18:10.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.table_properties_collectors: 2024-09-18T07:18:10.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.inplace_update_support: 0 2024-09-18T07:18:10.637 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-18T07:18:10.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-18T07:18:10.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-18T07:18:10.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-18T07:18:10.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.bloom_locality: 0 2024-09-18T07:18:10.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.max_successive_merges: 0 2024-09-18T07:18:10.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-18T07:18:10.638 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.paranoid_file_checks: 0 2024-09-18T07:18:10.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.force_consistency_checks: 1 2024-09-18T07:18:10.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.report_bg_io_stats: 0 2024-09-18T07:18:10.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.ttl: 2592000 2024-09-18T07:18:10.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-18T07:18:10.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.enable_blob_files: false 2024-09-18T07:18:10.640 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.min_blob_size: 0 2024-09-18T07:18:10.640 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.blob_file_size: 268435456 2024-09-18T07:18:10.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-18T07:18:10.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-18T07:18:10.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-18T07:18:10.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-18T07:18:10.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-18T07:18:10.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-18T07:18:10.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726643890375748, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-18T07:18:10.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726643890376493, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80060, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78323, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9946, "raw_average_key_size": 46, "raw_value_size": 72579, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726643890, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "2494b820-adfd-473a-a9f1-d827faedd810", "db_session_id": "COH9DDNB41WI1M9Z196I"}} 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726643890376860, "job": 1, "event": "recovery_finished"} 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi083/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55b9c40b0a80 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: DB pointer 0x55b9c40c0000 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** DB Stats ** 2024-09-18T07:18:10.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-18T07:18:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L0 2/0 79.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.6 0.00 0.00 1 0.000 0 0 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Sum 2/0 79.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.6 0.00 0.00 1 0.000 0 0 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.6 0.00 0.00 1 0.000 0 0 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 153.6 0.00 0.00 1 0.000 0 0 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-18T07:18:10.644 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative compaction: 0.00 GB write, 25.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval compaction: 0.00 GB write, 25.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-18T07:18:10.645 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L0 2/0 79.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.6 0.00 0.00 1 0.000 0 0 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Sum 2/0 79.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.6 0.00 0.00 1 0.000 0 0 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 153.6 0.00 0.00 1 0.000 0 0 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-18T07:18:10.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative compaction: 0.00 GB write, 25.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: starting mon.smithi083 rank 0 at public addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] at bind addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi083 fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mon.smithi083@-1(???) e1 preinit fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mon.smithi083@-1(???).mds e1 new map 2024-09-18T07:18:10.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mon.smithi083@-1(???).mds e1 print_map 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: e1 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: legacy client fscid: -1 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: No filesystems configured 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mon.smithi083@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mon.smithi083@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mon.smithi083@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mon.smithi083@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-18T07:18:10.648 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mon.smithi083@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta expand map: {default=false} 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta from 'false' to 'false' 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta expanded map: {default=false} 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta expand map: {default=info} 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta from 'info' to 'info' 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta expanded map: {default=info} 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta expand map: {default=daemon} 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta expanded map: {default=daemon} 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta expand map: {default=debug} 2024-09-18T07:18:10.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta from 'debug' to 'debug' 2024-09-18T07:18:10.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: expand_channel_meta expanded map: {default=debug} 2024-09-18T07:18:10.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 bash[23692]: 9f77aab8b41c29566ed04a6ad5d13afde2a31c1b8335c525b209784b131bddab 2024-09-18T07:18:10.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 systemd[1]: Started Ceph mon.smithi083 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:18:10.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mon.smithi083 is new leader, mons smithi083 in quorum (ranks 0) 2024-09-18T07:18:10.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: monmap e1: 1 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0]} 2024-09-18T07:18:10.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: fsmap 2024-09-18T07:18:10.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: osdmap e1: 0 total, 0 up, 0 in 2024-09-18T07:18:10.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:10 smithi083 ceph-mon[23711]: mgrmap e1: no daemons active 2024-09-18T07:18:11.390 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-18T07:18:11.391 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-18T07:18:11.391 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mgr... 2024-09-18T07:18:11.391 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying port 9283 ... 2024-09-18T07:18:11.558 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-126f471a-758e-11ef-bceb-c7b262605968@mgr.smithi083.vchvnu.service: Unit ceph-126f471a-758e-11ef-bceb-c7b262605968@mgr.smithi083.vchvnu.service not loaded. 2024-09-18T07:18:11.566 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-126f471a-758e-11ef-bceb-c7b262605968.target.wants/ceph-126f471a-758e-11ef-bceb-c7b262605968@mgr.smithi083.vchvnu.service → /etc/systemd/system/ceph-126f471a-758e-11ef-bceb-c7b262605968@.service. 2024-09-18T07:18:12.039 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:11 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2455071393' entity='client.admin' 2024-09-18T07:18:12.975 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-09-18T07:18:12.982 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-09-18T07:18:12.983 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-09-18T07:18:12.983 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2024-09-18T07:18:12.991 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-09-18T07:18:12.998 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-09-18T07:18:12.999 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-09-18T07:18:12.999 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-18T07:18:13.000 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr to start... 2024-09-18T07:18:13.000 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr... 2024-09-18T07:18:13.517 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:13.517 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "126f471a-758e-11ef-bceb-c7b262605968", 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "smithi083" 2024-09-18T07:18:13.518 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-18T07:18:13.519 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-09-18T07:18:13.520 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-09-18T07:18:13.521 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:13.522 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-09-18T07:18:13.522 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:13.522 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-09-18T07:17:55.270690+0000", 2024-09-18T07:18:13.522 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-09-18T07:18:13.522 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:13.522 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-18T07:18:13.522 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-09-18T07:18:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:13 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/902105438' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-18T07:18:14.040 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (1/15)... 2024-09-18T07:18:16.612 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:16.612 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-09-18T07:18:16.612 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "126f471a-758e-11ef-bceb-c7b262605968", 2024-09-18T07:18:16.612 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-09-18T07:18:16.612 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-18T07:18:16.612 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-09-18T07:18:16.612 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-09-18T07:18:16.612 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "smithi083" 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:16.613 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-18T07:18:16.614 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-09-18T07:18:16.615 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-09-18T07:17:55.270690+0000", 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-18T07:18:16.616 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-09-18T07:18:16.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:16 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1693658784' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-18T07:18:17.196 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (2/15)... 2024-09-18T07:18:18.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: Activating manager daemon smithi083.vchvnu 2024-09-18T07:18:18.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: mgrmap e2: smithi083.vchvnu(active, starting, since 0.00271647s) 2024-09-18T07:18:18.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-18T07:18:18.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-18T07:18:18.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-18T07:18:18.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:18:18.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:18:18.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: Manager daemon smithi083.vchvnu is now available 2024-09-18T07:18:18.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:18:18.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:18:18.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:18.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:18.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:18 smithi083 ceph-mon[23711]: from='mgr.14100 172.21.15.83:0/210075083' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "126f471a-758e-11ef-bceb-c7b262605968", 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-18T07:18:19.851 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "smithi083" 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-09-18T07:18:19.852 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-18T07:18:19.853 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-18T07:18:19.854 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-09-18T07:17:55.270690+0000", 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-09-18T07:18:19.855 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-09-18T07:18:19.856 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-18T07:18:19.856 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-09-18T07:18:20.409 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:20 smithi083 ceph-mon[23711]: mgrmap e3: smithi083.vchvnu(active, since 1.00676s) 2024-09-18T07:18:20.409 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:20 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/897241568' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-18T07:18:20.411 INFO:teuthology.orchestra.run.smithi083.stderr:mgr is available 2024-09-18T07:18:20.961 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:20.961 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-18T07:18:20.962 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-18T07:18:21.339 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:21 smithi083 ceph-mon[23711]: mgrmap e4: smithi083.vchvnu(active, since 2s) 2024-09-18T07:18:21.339 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:21 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1721744775' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-18T07:18:21.526 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling cephadm module... 2024-09-18T07:18:22.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:22 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3108451375' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-18T07:18:23.324 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:23 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3108451375' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-18T07:18:23.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:23 smithi083 ceph-mon[23711]: mgrmap e5: smithi083.vchvnu(active, since 4s) 2024-09-18T07:18:23.477 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-09-18T07:18:23.478 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-18T07:18:23.478 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-09-18T07:18:23.478 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "smithi083.vchvnu", 2024-09-18T07:18:23.478 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-18T07:18:23.478 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-09-18T07:18:24.032 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2024-09-18T07:18:24.032 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 5... 2024-09-18T07:18:24.323 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:24 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1658229323' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-18T07:18:29.244 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-09-18T07:18:29.244 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-18T07:18:29.244 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2024-09-18T07:18:29.244 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-09-18T07:18:29.478 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: Active manager daemon smithi083.vchvnu restarted 2024-09-18T07:18:29.478 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: Activating manager daemon smithi083.vchvnu 2024-09-18T07:18:29.478 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: osdmap e2: 0 total, 0 up, 0 in 2024-09-18T07:18:29.478 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: mgrmap e6: smithi083.vchvnu(active, starting, since 0.0527916s) 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: Manager daemon smithi083.vchvnu is now available 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:18:29.479 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:29 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:18:29.779 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 5 is available 2024-09-18T07:18:29.779 INFO:teuthology.orchestra.run.smithi083.stderr:Setting orchestrator backend to cephadm... 2024-09-18T07:18:30.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:30 smithi083 ceph-mon[23711]: mgrmap e7: smithi083.vchvnu(active, since 1.05498s) 2024-09-18T07:18:30.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:30 smithi083 ceph-mon[23711]: [18/Sep/2024:07:18:29] ENGINE Bus STARTING 2024-09-18T07:18:30.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:30 smithi083 ceph-mon[23711]: [18/Sep/2024:07:18:29] ENGINE Serving on https://172.21.15.83:7150 2024-09-18T07:18:30.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:30 smithi083 ceph-mon[23711]: [18/Sep/2024:07:18:29] ENGINE Bus STARTED 2024-09-18T07:18:30.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:30 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:30.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:30 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:18:31.499 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: value unchanged 2024-09-18T07:18:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:31 smithi083 ceph-mon[23711]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:31 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:31 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:18:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:31 smithi083 ceph-mon[23711]: mgrmap e8: smithi083.vchvnu(active, since 2s) 2024-09-18T07:18:32.092 INFO:teuthology.orchestra.run.smithi083.stderr:Generating ssh key... 2024-09-18T07:18:32.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:32 smithi083 ceph-mon[23711]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:33.985 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/Vv+xS/F5/cUo4SpLBEU1fp3AVRjT6m4+iExo4SPvgbwLq1nkZWb8RPK41bU/mZEr1VpGtgOLTzf00WC2WfhFhPB9GsvFTD4h8EDvZXwU4aalfPA0mEYVbPxHSFOzkePiXDbh/lF/Y9ZReRQzFrO5mCHTApZfKf7cEijyMj8lqGqYJmRn5yu2Ob/95KIDDEWfM853Rokr7Gnh3ylXFmhFVuWlQ/UbZNM9/bANn14x7/8U7NNjG1Dj5JfUMT8qpHXHqyG9d0QG02p1wrZcQbxJcx48DZR3jFUyLQeFfFadW3Bz8RHPrDqK0SEmVa1VsG7XugRPfeMEttU0iv6+8OnIhr8hjYUiLx0f4GmuqB6koNWsXr9JVCH1D+mNTXIUSNCCDN75D/9LcphEYhIOm+yAMjf+FgSvULLYCulFogyOjpvkNbg6p9ihpu7/wl8f6Ib4GBjfolQkXKQcjJdFoaLhT/okEUzXKpXfFYGr5PYk2lPIhEXT7p/o0fHfWbML8uE= ceph-126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:18:34.117 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:33 smithi083 ceph-mon[23711]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:34.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:33 smithi083 ceph-mon[23711]: Generating ssh key... 2024-09-18T07:18:34.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:33 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:34.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:33 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:34.538 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-18T07:18:34.539 INFO:teuthology.orchestra.run.smithi083.stderr:Adding key to root@localhost authorized_keys... 2024-09-18T07:18:34.543 INFO:teuthology.orchestra.run.smithi083.stderr:Adding host smithi083... 2024-09-18T07:18:35.117 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:34 smithi083 ceph-mon[23711]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:35.975 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Added host 'smithi083' with addr '172.21.15.83' 2024-09-18T07:18:35.997 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:35 smithi083 ceph-mon[23711]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi083", "addr": "172.21.15.83", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:36.555 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying mon service with default placement... 2024-09-18T07:18:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:36 smithi083 ceph-mon[23711]: Deploying cephadm binary to smithi083 2024-09-18T07:18:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:36 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:36 smithi083 ceph-mon[23711]: Added host smithi083 2024-09-18T07:18:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:36 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:18:37.148 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-18T07:18:37.745 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying mgr service with default placement... 2024-09-18T07:18:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:38 smithi083 ceph-mon[23711]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:38 smithi083 ceph-mon[23711]: Saving service mon spec with placement count:5 2024-09-18T07:18:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:38 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:38.513 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-18T07:18:39.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:39 smithi083 ceph-mon[23711]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:39.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:39 smithi083 ceph-mon[23711]: Saving service mgr spec with placement count:2 2024-09-18T07:18:39.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:39 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:39.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:39 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:39.761 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying crash service with default placement... 2024-09-18T07:18:40.388 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-18T07:18:41.092 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:40 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:41.093 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:40 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:41.362 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying prometheus service with default placement... 2024-09-18T07:18:41.997 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:41 smithi083 ceph-mon[23711]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:41.997 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:41 smithi083 ceph-mon[23711]: Saving service crash spec with placement * 2024-09-18T07:18:41.997 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:41 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:42.012 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-18T07:18:42.544 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying grafana service with default placement... 2024-09-18T07:18:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:43 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:43 smithi083 ceph-mon[23711]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:43 smithi083 ceph-mon[23711]: Saving service prometheus spec with placement count:1 2024-09-18T07:18:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:43 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:43.140 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-18T07:18:43.629 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying node-exporter service with default placement... 2024-09-18T07:18:44.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:44 smithi083 ceph-mon[23711]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:44.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:44 smithi083 ceph-mon[23711]: Saving service grafana spec with placement count:1 2024-09-18T07:18:44.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:44 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:44.232 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-18T07:18:45.296 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:45 smithi083 ceph-mon[23711]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:45.296 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:45 smithi083 ceph-mon[23711]: Saving service node-exporter spec with placement * 2024-09-18T07:18:45.296 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:45 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:45.299 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying alertmanager service with default placement... 2024-09-18T07:18:45.893 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-18T07:18:47.059 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:46 smithi083 ceph-mon[23711]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:47.059 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:46 smithi083 ceph-mon[23711]: Saving service alertmanager spec with placement count:1 2024-09-18T07:18:47.059 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:46 smithi083 ceph-mon[23711]: from='mgr.14118 172.21.15.83:0/1157256714' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:48.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:48 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/140188666' entity='client.admin' 2024-09-18T07:18:48.835 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling the dashboard module... 2024-09-18T07:18:49.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:49 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1279838228' entity='client.admin' 2024-09-18T07:18:50.442 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:50 smithi083 ceph-mon[23711]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:18:50.443 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:50 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3525411984' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-18T07:18:51.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:51 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3525411984' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-18T07:18:51.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:51 smithi083 ceph-mon[23711]: mgrmap e9: smithi083.vchvnu(active, since 22s) 2024-09-18T07:18:51.427 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-09-18T07:18:51.427 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-18T07:18:51.427 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-09-18T07:18:51.428 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "smithi083.vchvnu", 2024-09-18T07:18:51.428 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-18T07:18:51.428 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-09-18T07:18:51.932 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2024-09-18T07:18:51.933 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 9... 2024-09-18T07:18:52.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:52 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1354537906' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-18T07:18:56.492 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:56 smithi083 ceph-mon[23711]: Active manager daemon smithi083.vchvnu restarted 2024-09-18T07:18:56.492 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:56 smithi083 ceph-mon[23711]: Activating manager daemon smithi083.vchvnu 2024-09-18T07:18:56.492 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:56 smithi083 ceph-mon[23711]: osdmap e3: 0 total, 0 up, 0 in 2024-09-18T07:18:57.412 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-09-18T07:18:57.413 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-18T07:18:57.413 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2024-09-18T07:18:57.413 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-09-18T07:18:57.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: mgrmap e10: smithi083.vchvnu(active, starting, since 0.0711769s) 2024-09-18T07:18:57.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:18:57.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:18:57.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-18T07:18:57.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-18T07:18:57.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-18T07:18:57.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: Manager daemon smithi083.vchvnu is now available 2024-09-18T07:18:57.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:18:57.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:18:57.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:18:57.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:18:57.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: [18/Sep/2024:07:18:56] ENGINE Bus STARTING 2024-09-18T07:18:57.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: [18/Sep/2024:07:18:57] ENGINE Serving on https://172.21.15.83:7150 2024-09-18T07:18:57.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: [18/Sep/2024:07:18:57] ENGINE Bus STARTED 2024-09-18T07:18:57.457 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:57 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:58.056 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 9 is available 2024-09-18T07:18:58.056 INFO:teuthology.orchestra.run.smithi083.stderr:Generating a dashboard self-signed certificate... 2024-09-18T07:18:58.576 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:58 smithi083 ceph-mon[23711]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-18T07:18:58.576 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:58 smithi083 ceph-mon[23711]: mgrmap e11: smithi083.vchvnu(active, since 1.07597s) 2024-09-18T07:18:58.576 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:58 smithi083 ceph-mon[23711]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-18T07:18:58.775 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-18T07:18:59.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:59 smithi083 ceph-mon[23711]: mgrmap e12: smithi083.vchvnu(active, since 2s) 2024-09-18T07:18:59.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:59 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:59.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:59 smithi083 ceph-mon[23711]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:18:59.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:59 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:59.456 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:18:59 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:18:59.457 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial admin user... 2024-09-18T07:19:00.343 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$kVj86xcNn1qHwB40ab3p1Oil1L1vPzqGCfT61bQhdwAeGE1WvwBsm", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726643940, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-18T07:19:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:00 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:00 smithi083 ceph-mon[23711]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:19:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:00 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:01.463 INFO:teuthology.orchestra.run.smithi083.stderr:Fetching dashboard port number... 2024-09-18T07:19:02.098 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 8443 2024-09-18T07:19:02.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:02 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3708003327' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-18T07:19:02.688 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-09-18T07:19:02.694 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-09-18T07:19:02.695 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-09-18T07:19:02.695 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-18T07:19:02.697 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph Dashboard is now available at: 2024-09-18T07:19:02.697 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:19:02.697 INFO:teuthology.orchestra.run.smithi083.stderr: URL: https://smithi083.front.sepia.ceph.com:8443/ 2024-09-18T07:19:02.698 INFO:teuthology.orchestra.run.smithi083.stderr: User: admin 2024-09-18T07:19:02.698 INFO:teuthology.orchestra.run.smithi083.stderr: Password: 1nfz7r45r9 2024-09-18T07:19:02.698 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:19:02.698 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling autotune for osd_memory_target 2024-09-18T07:19:04.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:04 smithi083 ceph-mon[23711]: mgrmap e13: smithi083.vchvnu(active, since 6s) 2024-09-18T07:19:04.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-18T07:19:05.017 INFO:teuthology.orchestra.run.smithi083.stderr:You can access the Ceph CLI with: 2024-09-18T07:19:05.017 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:19:05.017 INFO:teuthology.orchestra.run.smithi083.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-18T07:19:05.017 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:19:05.017 INFO:teuthology.orchestra.run.smithi083.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-18T07:19:05.017 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:19:05.017 INFO:teuthology.orchestra.run.smithi083.stderr: ceph telemetry on 2024-09-18T07:19:05.017 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:19:05.017 INFO:teuthology.orchestra.run.smithi083.stderr:For more information see: 2024-09-18T07:19:05.018 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:19:05.018 INFO:teuthology.orchestra.run.smithi083.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-18T07:19:05.018 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:19:05.018 INFO:teuthology.orchestra.run.smithi083.stderr:Bootstrap complete. 2024-09-18T07:19:05.050 INFO:tasks.cephadm:Fetching config... 2024-09-18T07:19:05.051 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:19:05.051 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-18T07:19:05.067 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-18T07:19:05.067 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:19:05.068 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-18T07:19:05.123 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-18T07:19:05.123 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:19:05.123 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/keyring of=/dev/stdout 2024-09-18T07:19:05.188 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-18T07:19:05.189 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:19:05.189 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-18T07:19:05.244 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-18T07:19:05.244 DEBUG:teuthology.orchestra.run.smithi083:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/Vv+xS/F5/cUo4SpLBEU1fp3AVRjT6m4+iExo4SPvgbwLq1nkZWb8RPK41bU/mZEr1VpGtgOLTzf00WC2WfhFhPB9GsvFTD4h8EDvZXwU4aalfPA0mEYVbPxHSFOzkePiXDbh/lF/Y9ZReRQzFrO5mCHTApZfKf7cEijyMj8lqGqYJmRn5yu2Ob/95KIDDEWfM853Rokr7Gnh3ylXFmhFVuWlQ/UbZNM9/bANn14x7/8U7NNjG1Dj5JfUMT8qpHXHqyG9d0QG02p1wrZcQbxJcx48DZR3jFUyLQeFfFadW3Bz8RHPrDqK0SEmVa1VsG7XugRPfeMEttU0iv6+8OnIhr8hjYUiLx0f4GmuqB6koNWsXr9JVCH1D+mNTXIUSNCCDN75D/9LcphEYhIOm+yAMjf+FgSvULLYCulFogyOjpvkNbg6p9ihpu7/wl8f6Ib4GBjfolQkXKQcjJdFoaLhT/okEUzXKpXfFYGr5PYk2lPIhEXT7p/o0fHfWbML8uE= ceph-126f471a-758e-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-18T07:19:05.320 INFO:teuthology.orchestra.run.smithi083.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/Vv+xS/F5/cUo4SpLBEU1fp3AVRjT6m4+iExo4SPvgbwLq1nkZWb8RPK41bU/mZEr1VpGtgOLTzf00WC2WfhFhPB9GsvFTD4h8EDvZXwU4aalfPA0mEYVbPxHSFOzkePiXDbh/lF/Y9ZReRQzFrO5mCHTApZfKf7cEijyMj8lqGqYJmRn5yu2Ob/95KIDDEWfM853Rokr7Gnh3ylXFmhFVuWlQ/UbZNM9/bANn14x7/8U7NNjG1Dj5JfUMT8qpHXHqyG9d0QG02p1wrZcQbxJcx48DZR3jFUyLQeFfFadW3Bz8RHPrDqK0SEmVa1VsG7XugRPfeMEttU0iv6+8OnIhr8hjYUiLx0f4GmuqB6koNWsXr9JVCH1D+mNTXIUSNCCDN75D/9LcphEYhIOm+yAMjf+FgSvULLYCulFogyOjpvkNbg6p9ihpu7/wl8f6Ib4GBjfolQkXKQcjJdFoaLhT/okEUzXKpXfFYGr5PYk2lPIhEXT7p/o0fHfWbML8uE= ceph-126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:19:05.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/Vv+xS/F5/cUo4SpLBEU1fp3AVRjT6m4+iExo4SPvgbwLq1nkZWb8RPK41bU/mZEr1VpGtgOLTzf00WC2WfhFhPB9GsvFTD4h8EDvZXwU4aalfPA0mEYVbPxHSFOzkePiXDbh/lF/Y9ZReRQzFrO5mCHTApZfKf7cEijyMj8lqGqYJmRn5yu2Ob/95KIDDEWfM853Rokr7Gnh3ylXFmhFVuWlQ/UbZNM9/bANn14x7/8U7NNjG1Dj5JfUMT8qpHXHqyG9d0QG02p1wrZcQbxJcx48DZR3jFUyLQeFfFadW3Bz8RHPrDqK0SEmVa1VsG7XugRPfeMEttU0iv6+8OnIhr8hjYUiLx0f4GmuqB6koNWsXr9JVCH1D+mNTXIUSNCCDN75D/9LcphEYhIOm+yAMjf+FgSvULLYCulFogyOjpvkNbg6p9ihpu7/wl8f6Ib4GBjfolQkXKQcjJdFoaLhT/okEUzXKpXfFYGr5PYk2lPIhEXT7p/o0fHfWbML8uE= ceph-126f471a-758e-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-18T07:19:05.398 INFO:teuthology.orchestra.run.smithi096.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/Vv+xS/F5/cUo4SpLBEU1fp3AVRjT6m4+iExo4SPvgbwLq1nkZWb8RPK41bU/mZEr1VpGtgOLTzf00WC2WfhFhPB9GsvFTD4h8EDvZXwU4aalfPA0mEYVbPxHSFOzkePiXDbh/lF/Y9ZReRQzFrO5mCHTApZfKf7cEijyMj8lqGqYJmRn5yu2Ob/95KIDDEWfM853Rokr7Gnh3ylXFmhFVuWlQ/UbZNM9/bANn14x7/8U7NNjG1Dj5JfUMT8qpHXHqyG9d0QG02p1wrZcQbxJcx48DZR3jFUyLQeFfFadW3Bz8RHPrDqK0SEmVa1VsG7XugRPfeMEttU0iv6+8OnIhr8hjYUiLx0f4GmuqB6koNWsXr9JVCH1D+mNTXIUSNCCDN75D/9LcphEYhIOm+yAMjf+FgSvULLYCulFogyOjpvkNbg6p9ihpu7/wl8f6Ib4GBjfolQkXKQcjJdFoaLhT/okEUzXKpXfFYGr5PYk2lPIhEXT7p/o0fHfWbML8uE= ceph-126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:19:05.414 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-18T07:19:05.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:05 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3046159222' entity='client.admin' 2024-09-18T07:19:08.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:07 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:08.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:07 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:19:08.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:07 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:08.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:07 smithi083 ceph-mon[23711]: Deploying daemon alertmanager.smithi083 on smithi083 2024-09-18T07:19:08.629 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-18T07:19:08.629 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-18T07:19:09.217 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:08 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2034633339' entity='client.admin' 2024-09-18T07:19:11.788 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi096 2024-09-18T07:19:11.788 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-18T07:19:11.789 DEBUG:teuthology.orchestra.run.smithi096:> dd of=/etc/ceph/ceph.conf 2024-09-18T07:19:11.808 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-18T07:19:11.808 DEBUG:teuthology.orchestra.run.smithi096:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:19:11.867 INFO:tasks.cephadm:Adding host smithi096 to orchestrator... 2024-09-18T07:19:11.868 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch host add smithi096 2024-09-18T07:19:12.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:12 smithi083 ceph-mon[23711]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:19:12.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:12 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:15.206 INFO:teuthology.orchestra.run.smithi083.stdout:Added host 'smithi096' with addr '172.21.15.96' 2024-09-18T07:19:16.591 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:16 smithi083 ceph-mon[23711]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi096", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:19:16.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:16 smithi083 ceph-mon[23711]: Deploying cephadm binary to smithi096 2024-09-18T07:19:16.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:16 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:16.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:16 smithi083 ceph-mon[23711]: Added host smithi096 2024-09-18T07:19:18.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:18 smithi083 ceph-mon[23711]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:18.398 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-18T07:19:20.302 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:20 smithi083 ceph-mon[23711]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:20.303 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:20 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:20.303 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:20 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-18T07:19:20.303 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:20 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-18T07:19:20.303 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:20 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:19:20.303 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:20 smithi083 ceph-mon[23711]: Deploying daemon crash.smithi083 on smithi083 2024-09-18T07:19:21.187 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:19:21.188 INFO:teuthology.orchestra.run.smithi083.stdout:[{"addr": "172.21.15.83", "hostname": "smithi083", "labels": [], "status": ""}, {"addr": "172.21.15.96", "hostname": "smithi096", "labels": [], "status": ""}] 2024-09-18T07:19:21.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:21 smithi083 ceph-mon[23711]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:21.689 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:21 smithi083 ceph-mon[23711]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:19:22.460 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:22 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:22.462 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-18T07:19:22.462 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-18T07:19:24.304 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:24 smithi083 ceph-mon[23711]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:24.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:24 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:24.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:24 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:24.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:24 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:24.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:24 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-18T07:19:24.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:24 smithi083 ceph-mon[23711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-18T07:19:24.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:24 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:24.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:24 smithi083 ceph-mon[23711]: Deploying daemon grafana.smithi083 on smithi083 2024-09-18T07:19:25.104 INFO:teuthology.orchestra.run.smithi083.stderr:adjusted tunables profile to default 2024-09-18T07:19:25.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:25 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3105159702' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-18T07:19:25.736 INFO:tasks.cephadm:Adding mon.smithi083 on smithi083 2024-09-18T07:19:25.736 INFO:tasks.cephadm:Adding mon.smithi096 on smithi096 2024-09-18T07:19:25.736 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi083:172.21.15.83=smithi083;smithi096:172.21.15.96=smithi096' 2024-09-18T07:19:26.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:26 smithi083 ceph-mon[23711]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:26.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:26 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3105159702' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-18T07:19:26.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:26 smithi083 ceph-mon[23711]: osdmap e4: 0 total, 0 up, 0 in 2024-09-18T07:19:27.519 INFO:teuthology.orchestra.run.smithi096.stdout:Scheduled mon update... 2024-09-18T07:19:27.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:27 smithi083 ceph-mon[23711]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:27.749 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:27 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:28.079 DEBUG:teuthology.orchestra.run.smithi096:mon.smithi096> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi096.service 2024-09-18T07:19:28.082 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:19:28.082 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:19:28.617 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:28 smithi083 ceph-mon[23711]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi083:172.21.15.83=smithi083;smithi096:172.21.15.96=smithi096", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:19:28.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:28 smithi083 ceph-mon[23711]: Saving service mon spec with placement smithi083:172.21.15.83=smithi083;smithi096:172.21.15.96=smithi096;count:2 2024-09-18T07:19:28.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:28 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:29.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:29 smithi083 ceph-mon[23711]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:29.878 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:19:29.878 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:19:29.880 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:19:30.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:30 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/252406440' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:19:31.466 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:19:31.467 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:19:31.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:31 smithi083 ceph-mon[23711]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:33.322 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:19:33.322 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:19:33.324 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:19:33.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:33 smithi083 ceph-mon[23711]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:33.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:33 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/1232021700' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:19:34.853 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:19:34.853 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:19:35.107 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:34 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:35.107 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:34 smithi083 ceph-mon[23711]: Deploying daemon node-exporter.smithi083 on smithi083 2024-09-18T07:19:36.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:35 smithi083 ceph-mon[23711]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:36.621 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:19:36.621 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:19:36.623 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:19:37.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:37 smithi083 ceph-mon[23711]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:19:37.803 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:37 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:37.803 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:37 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/725845970' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:19:37.803 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:37 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:37.803 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:37 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-18T07:19:38.208 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:19:38.208 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:19:38.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:38 smithi083 ceph-mon[23711]: from='mgr.14160 172.21.15.83:0/143582522' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-18T07:19:38.803 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:38 smithi083 ceph-mon[23711]: mgrmap e14: smithi083.vchvnu(active, since 41s) 2024-09-18T07:19:40.027 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:19:40.027 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:19:40.028 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:19:40.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:40 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/2465325487' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:19:41.620 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:19:41.620 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:19:43.433 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:19:43.434 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:19:43.435 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:19:43.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:43 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/1260520267' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:19:44.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:44 smithi083 ceph-mon[23711]: Active manager daemon smithi083.vchvnu restarted 2024-09-18T07:19:44.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:44 smithi083 ceph-mon[23711]: Activating manager daemon smithi083.vchvnu 2024-09-18T07:19:44.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:44 smithi083 ceph-mon[23711]: osdmap e5: 0 total, 0 up, 0 in 2024-09-18T07:19:45.106 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:19:45.107 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:19:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: mgrmap e15: smithi083.vchvnu(active, starting, since 0.841804s) 2024-09-18T07:19:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:19:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:19:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-18T07:19:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-18T07:19:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-18T07:19:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: Manager daemon smithi083.vchvnu is now available 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:45.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:46.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:46 smithi083 ceph-mon[23711]: [18/Sep/2024:07:19:44] ENGINE Bus STARTING 2024-09-18T07:19:46.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:46 smithi083 ceph-mon[23711]: [18/Sep/2024:07:19:44] ENGINE Serving on https://172.21.15.83:7150 2024-09-18T07:19:46.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:46 smithi083 ceph-mon[23711]: [18/Sep/2024:07:19:44] ENGINE Bus STARTED 2024-09-18T07:19:46.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:46 smithi083 ceph-mon[23711]: mgrmap e16: smithi083.vchvnu(active, since 1.84464s) 2024-09-18T07:19:46.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:47.119 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:19:47.119 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:19:47.121 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:19:47.617 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:47 smithi083 ceph-mon[23711]: mgrmap e17: smithi083.vchvnu(active, since 2s) 2024-09-18T07:19:47.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:47 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/2253465065' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:19:48.712 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:19:48.713 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:19:48.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:48 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:48.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:48 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:48.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:48 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:19:48.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:48 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:50.117 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:49 smithi083 ceph-mon[23711]: Updating smithi083:/etc/ceph/ceph.conf 2024-09-18T07:19:50.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:49 smithi083 ceph-mon[23711]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:19:51.302 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:19:51.302 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:19:51.304 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:19:52.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:52.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:52.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd/host:smithi096", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:19:52.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:52.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi096", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-18T07:19:52.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi096", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-18T07:19:52.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:19:52.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:51 smithi083 ceph-mon[23711]: Deploying daemon crash.smithi096 on smithi096 2024-09-18T07:19:52.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:51 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/3051630263' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:19:53.149 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:19:53.150 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:19:55.112 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:19:55.112 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:19:55.113 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:19:55.617 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:55 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/2440249104' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:19:56.862 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:56 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:19:56.862 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:56 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:19:56.863 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:56 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-18T07:19:56.863 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:56 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:19:56.863 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:56 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:19:56.863 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:56 smithi083 ceph-mon[23711]: Deploying daemon mgr.smithi096.caifnv on smithi096 2024-09-18T07:19:56.998 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:19:56.998 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:19:58.847 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:19:58.848 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:19:58.849 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:19:59.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:19:58 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/184060988' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:20:00.401 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:20:00.401 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:20:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-18T07:20:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:00 smithi083 ceph-mon[23711]: Deploying daemon mon.smithi096 on smithi096 2024-09-18T07:20:02.279 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:20:02.279 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":1,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:17:52.460906Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-18T07:20:02.281 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 1 2024-09-18T07:20:02.617 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:02 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/1667351074' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:20:04.027 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-18T07:20:04.028 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-18T07:20:05.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:05 smithi083 ceph-mon[23711]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:05.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:05 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:11.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:20:11.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: mon.smithi083 calling monitor election 2024-09-18T07:20:11.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:20:11.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:11.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: mon.smithi096 calling monitor election 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: mon.smithi083 is new leader, mons smithi083,smithi096 in quorum (ranks 0,1) 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: monmap e2: 2 mons at {smithi083=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],smithi096=[v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0]} 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: fsmap 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: osdmap e5: 0 total, 0 up, 0 in 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: mgrmap e17: smithi083.vchvnu(active, since 27s) 2024-09-18T07:20:11.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:20:11.370 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: overall HEALTH_OK 2024-09-18T07:20:11.370 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:12.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:11 smithi083 ceph-mon[23711]: Deploying daemon node-exporter.smithi096 on smithi096 2024-09-18T07:20:12.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:11 smithi083 ceph-mon[23711]: mgrmap e18: smithi083.vchvnu(active, since 27s), standbys: smithi096.caifnv 2024-09-18T07:20:12.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:11 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr metadata", "who": "smithi096.caifnv", "id": "smithi096.caifnv"}]: dispatch 2024-09-18T07:20:12.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:11 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:20:12.548 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:20:12.549 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":2,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","modified":"2024-09-18T07:20:05.813849Z","created":"2024-09-18T07:17:52.460906Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi083","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi096","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:3300","nonce":0},{"type":"v1","addr":"172.21.15.96:6789","nonce":0}]},"addr":"172.21.15.96:6789/0","public_addr":"172.21.15.96:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-18T07:20:12.550 INFO:teuthology.orchestra.run.smithi096.stderr:dumped monmap epoch 2 2024-09-18T07:20:13.327 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:12 smithi096 ceph-mon[25928]: from='client.? 172.21.15.96:0/2813407864' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:20:13.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:12 smithi083 ceph-mon[23711]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:13.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:12 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/2813407864' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-18T07:20:14.013 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-18T07:20:14.014 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-18T07:20:15.844 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:15.845 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:15 smithi083 ceph-mon[23711]: Deploying daemon prometheus.smithi083 on smithi083 2024-09-18T07:20:15.845 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:15 smithi083 ceph-mon[23711]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:15.845 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:15.845 INFO:teuthology.orchestra.run.smithi083.stdout:# minimal ceph.conf for 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:20:15.845 INFO:teuthology.orchestra.run.smithi083.stdout:[global] 2024-09-18T07:20:15.845 INFO:teuthology.orchestra.run.smithi083.stdout: fsid = 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:20:15.845 INFO:teuthology.orchestra.run.smithi083.stdout: mon_host = [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] 2024-09-18T07:20:15.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:15 smithi096 ceph-mon[25928]: Deploying daemon prometheus.smithi083 on smithi083 2024-09-18T07:20:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:15 smithi096 ceph-mon[25928]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:16.578 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-18T07:20:16.578 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:20:16.578 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.conf 2024-09-18T07:20:16.605 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:20:16.605 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:20:16.670 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-18T07:20:16.670 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/ceph/ceph.conf 2024-09-18T07:20:16.703 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-18T07:20:16.703 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:20:16.773 DEBUG:tasks.cephadm:set 0 configs 2024-09-18T07:20:16.774 INFO:tasks.cephadm:Deploying OSDs... 2024-09-18T07:20:16.774 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:20:16.774 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/scratch_devs of=/dev/stdout 2024-09-18T07:20:16.791 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-18T07:20:16.791 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_1 2024-09-18T07:20:16.828 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:16 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2256838189' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:16.828 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:16 smithi083 ceph-mon[23711]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:16.845 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-18T07:20:16.846 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T07:20:16.846 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-18T07:20:16.846 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T07:20:16.846 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T07:20:16.846 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-09-18 07:19:34.472687107 +0000 2024-09-18T07:20:16.846 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-09-18 07:17:11.990936016 +0000 2024-09-18T07:20:16.846 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-09-18 07:17:11.990936016 +0000 2024-09-18T07:20:16.846 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-09-18 07:17:11.990936016 +0000 2024-09-18T07:20:16.846 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-18T07:20:16.915 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-09-18T07:20:16.915 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-09-18T07:20:16.915 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000208704 s, 2.5 MB/s 2024-09-18T07:20:16.916 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-18T07:20:16.975 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_2 2024-09-18T07:20:17.035 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-18T07:20:17.035 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T07:20:17.036 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-18T07:20:17.036 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T07:20:17.036 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T07:20:17.036 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-09-18 07:19:34.472687107 +0000 2024-09-18T07:20:17.036 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-09-18 07:17:12.321928089 +0000 2024-09-18T07:20:17.036 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-09-18 07:17:12.321928089 +0000 2024-09-18T07:20:17.036 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-09-18 07:17:12.321928089 +0000 2024-09-18T07:20:17.036 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-18T07:20:17.110 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-09-18T07:20:17.111 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-09-18T07:20:17.111 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.00019291 s, 2.7 MB/s 2024-09-18T07:20:17.112 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-18T07:20:17.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:16 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2256838189' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:17.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:16 smithi096 ceph-mon[25928]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:17.171 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_3 2024-09-18T07:20:17.226 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-18T07:20:17.226 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T07:20:17.226 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-18T07:20:17.226 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T07:20:17.227 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T07:20:17.227 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-09-18 07:19:34.472687107 +0000 2024-09-18T07:20:17.227 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-09-18 07:17:12.653920138 +0000 2024-09-18T07:20:17.227 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-09-18 07:17:12.653920138 +0000 2024-09-18T07:20:17.227 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-09-18 07:17:12.653920138 +0000 2024-09-18T07:20:17.227 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-18T07:20:17.289 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-09-18T07:20:17.289 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-09-18T07:20:17.289 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000190394 s, 2.7 MB/s 2024-09-18T07:20:17.290 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-18T07:20:17.349 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_4 2024-09-18T07:20:17.404 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-18T07:20:17.404 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T07:20:17.404 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-18T07:20:17.404 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T07:20:17.404 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T07:20:17.404 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-09-18 07:19:34.473687085 +0000 2024-09-18T07:20:17.404 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-09-18 07:17:12.970912546 +0000 2024-09-18T07:20:17.404 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-09-18 07:17:12.970912546 +0000 2024-09-18T07:20:17.405 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-09-18 07:17:12.970912546 +0000 2024-09-18T07:20:17.405 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-18T07:20:17.467 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-09-18T07:20:17.467 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-09-18T07:20:17.467 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000174197 s, 2.9 MB/s 2024-09-18T07:20:17.468 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-18T07:20:17.524 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2024-09-18T07:20:17.524 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/scratch_devs of=/dev/stdout 2024-09-18T07:20:17.544 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-18T07:20:17.544 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_1 2024-09-18T07:20:17.603 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-18T07:20:17.603 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T07:20:17.604 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-18T07:20:17.604 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T07:20:17.604 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T07:20:17.604 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2024-09-18 07:20:11.403091448 +0000 2024-09-18T07:20:17.604 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2024-09-18 07:17:12.191139930 +0000 2024-09-18T07:20:17.604 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2024-09-18 07:17:12.191139930 +0000 2024-09-18T07:20:17.604 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: 2024-09-18 07:17:12.191139930 +0000 2024-09-18T07:20:17.604 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-18T07:20:17.672 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2024-09-18T07:20:17.672 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2024-09-18T07:20:17.672 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.00027147 s, 1.9 MB/s 2024-09-18T07:20:17.673 DEBUG:teuthology.orchestra.run.smithi096:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-18T07:20:17.732 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_2 2024-09-18T07:20:17.790 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-18T07:20:17.790 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T07:20:17.790 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-18T07:20:17.790 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T07:20:17.790 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T07:20:17.790 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2024-09-18 07:20:11.403091448 +0000 2024-09-18T07:20:17.790 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2024-09-18 07:17:12.506132825 +0000 2024-09-18T07:20:17.790 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2024-09-18 07:17:12.506132825 +0000 2024-09-18T07:20:17.790 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: 2024-09-18 07:17:12.506132825 +0000 2024-09-18T07:20:17.791 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-18T07:20:17.855 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2024-09-18T07:20:17.855 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2024-09-18T07:20:17.855 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000235839 s, 2.2 MB/s 2024-09-18T07:20:17.856 DEBUG:teuthology.orchestra.run.smithi096:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-18T07:20:17.914 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_3 2024-09-18T07:20:17.971 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-18T07:20:17.971 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T07:20:17.971 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-18T07:20:17.971 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T07:20:17.971 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T07:20:17.971 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2024-09-18 07:20:11.403091448 +0000 2024-09-18T07:20:17.971 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2024-09-18 07:17:12.840125291 +0000 2024-09-18T07:20:17.971 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2024-09-18 07:17:12.840125291 +0000 2024-09-18T07:20:17.971 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: 2024-09-18 07:17:12.840125291 +0000 2024-09-18T07:20:17.972 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-18T07:20:18.035 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2024-09-18T07:20:18.035 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2024-09-18T07:20:18.035 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000251104 s, 2.0 MB/s 2024-09-18T07:20:18.036 DEBUG:teuthology.orchestra.run.smithi096:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-18T07:20:18.093 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_4 2024-09-18T07:20:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-18T07:20:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T07:20:18.150 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-18T07:20:18.150 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T07:20:18.150 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T07:20:18.150 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2024-09-18 07:20:11.404091426 +0000 2024-09-18T07:20:18.150 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2024-09-18 07:17:13.170117847 +0000 2024-09-18T07:20:18.150 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2024-09-18 07:17:13.170117847 +0000 2024-09-18T07:20:18.150 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: 2024-09-18 07:17:13.170117847 +0000 2024-09-18T07:20:18.150 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-18T07:20:18.213 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2024-09-18T07:20:18.213 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2024-09-18T07:20:18.213 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000199184 s, 2.6 MB/s 2024-09-18T07:20:18.214 DEBUG:teuthology.orchestra.run.smithi096:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-18T07:20:18.271 INFO:tasks.cephadm:Deploying osd.0 on smithi083 with /dev/vg_nvme/lv_4... 2024-09-18T07:20:18.271 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-18T07:20:19.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:19 smithi083 ceph-mon[23711]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:19.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:19 smithi096 ceph-mon[25928]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:21.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:21 smithi083 ceph-mon[23711]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:21.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:21 smithi096 ceph-mon[25928]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:23.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:23 smithi083 ceph-mon[23711]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:23.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:23 smithi096 ceph-mon[25928]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:25.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:25 smithi083 ceph-mon[23711]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:25.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:25 smithi096 ceph-mon[25928]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:27.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:27 smithi083 ceph-mon[23711]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:27.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:27.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:20:27.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:27.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:20:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:27 smithi096 ceph-mon[25928]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:20:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:20:28.467 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:20:28.489 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi083:vg_nvme/lv_4 2024-09-18T07:20:29.502 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:29.502 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:29 smithi083 ceph-mon[23711]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-18T07:20:29.502 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:29 smithi083 ceph-mon[23711]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:29.502 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:29 smithi083 ceph-mon[23711]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:20:29.502 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:29.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:29.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:29 smithi096 ceph-mon[25928]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-18T07:20:29.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:29 smithi096 ceph-mon[25928]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:29.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:29 smithi096 ceph-mon[25928]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:20:29.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:30.710 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:30 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:30.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:30 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:31 smithi083 ceph-mon[23711]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:31 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:20:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:31 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:20:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:31 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:31.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:31 smithi096 ceph-mon[25928]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:31.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:31 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:20:31.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:31 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:20:31.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:31 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:32.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:32 smithi083 ceph-mon[23711]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:20:32.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:32 smithi096 ceph-mon[25928]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:20:33.817 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:33 smithi083 ceph-mon[23711]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:33.817 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:33 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:33.817 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:33 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:33.817 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:33 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:33.817 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:33 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-18T07:20:33.817 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:33 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-18T07:20:33.817 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:33 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:33.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:33 smithi096 ceph-mon[25928]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:33.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:33 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:33.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:33 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:33.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:33 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:33.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:33 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-18T07:20:33.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:33 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-18T07:20:33.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:33 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:34.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:34 smithi083 ceph-mon[23711]: Updating smithi083:/etc/ceph/ceph.conf 2024-09-18T07:20:34.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:34 smithi083 ceph-mon[23711]: Reconfiguring mon.smithi083 (unknown last config time)... 2024-09-18T07:20:34.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:34 smithi083 ceph-mon[23711]: Reconfiguring daemon mon.smithi083 on smithi083 2024-09-18T07:20:34.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:34 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1760719019' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb38d4aa-4e71-4a92-9d7e-a3f5df569ab1"}]: dispatch 2024-09-18T07:20:34.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:34 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1760719019' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb38d4aa-4e71-4a92-9d7e-a3f5df569ab1"}]': finished 2024-09-18T07:20:34.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:34 smithi083 ceph-mon[23711]: osdmap e6: 1 total, 0 up, 1 in 2024-09-18T07:20:34.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:34 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:34.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:34 smithi096 ceph-mon[25928]: Updating smithi083:/etc/ceph/ceph.conf 2024-09-18T07:20:34.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:34 smithi096 ceph-mon[25928]: Reconfiguring mon.smithi083 (unknown last config time)... 2024-09-18T07:20:34.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:34 smithi096 ceph-mon[25928]: Reconfiguring daemon mon.smithi083 on smithi083 2024-09-18T07:20:34.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:34 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1760719019' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb38d4aa-4e71-4a92-9d7e-a3f5df569ab1"}]: dispatch 2024-09-18T07:20:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:34 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1760719019' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb38d4aa-4e71-4a92-9d7e-a3f5df569ab1"}]': finished 2024-09-18T07:20:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:34 smithi096 ceph-mon[25928]: osdmap e6: 1 total, 0 up, 1 in 2024-09-18T07:20:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:34 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:35.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:35 smithi096 ceph-mon[25928]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:35.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:35 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/779222537' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:20:35.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:35 smithi083 ceph-mon[23711]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:35.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:35 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/779222537' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:20:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:37 smithi083 ceph-mon[23711]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:37.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:37 smithi096 ceph-mon[25928]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:39.813 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:39 smithi083 ceph-mon[23711]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:39.813 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:39.813 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.vchvnu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:20:39.814 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:20:39.814 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:39 smithi096 ceph-mon[25928]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.vchvnu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:20:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:20:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:40.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:40 smithi083 ceph-mon[23711]: Reconfiguring mgr.smithi083.vchvnu (unknown last config time)... 2024-09-18T07:20:40.662 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:40 smithi083 ceph-mon[23711]: Reconfiguring daemon mgr.smithi083.vchvnu on smithi083 2024-09-18T07:20:40.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:40 smithi096 ceph-mon[25928]: Reconfiguring mgr.smithi083.vchvnu (unknown last config time)... 2024-09-18T07:20:40.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:40 smithi096 ceph-mon[25928]: Reconfiguring daemon mgr.smithi083.vchvnu on smithi083 2024-09-18T07:20:41.821 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:41 smithi083 ceph-mon[23711]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:41.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:41 smithi096 ceph-mon[25928]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:43.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:42 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:43.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:42 smithi083 ceph-mon[23711]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-09-18T07:20:43.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:42 smithi083 ceph-mon[23711]: Reconfiguring daemon alertmanager.smithi083 on smithi083 2024-09-18T07:20:43.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:42 smithi083 ceph-mon[23711]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:42 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:42 smithi096 ceph-mon[25928]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-09-18T07:20:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:42 smithi096 ceph-mon[25928]: Reconfiguring daemon alertmanager.smithi083 on smithi083 2024-09-18T07:20:43.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:42 smithi096 ceph-mon[25928]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-18T07:20:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:43 smithi083 ceph-mon[23711]: Deploying daemon osd.0 on smithi083 2024-09-18T07:20:44.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-18T07:20:44.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:44.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:43 smithi096 ceph-mon[25928]: Deploying daemon osd.0 on smithi083 2024-09-18T07:20:44.964 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:44 smithi083 ceph-mon[23711]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:44.964 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:20:44.964 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:20:45.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:44 smithi096 ceph-mon[25928]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:45.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:20:45.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:20:47.587 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:47 smithi083 ceph-mon[23711]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:47.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:47 smithi096 ceph-mon[25928]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:48.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:48 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:48.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:48 smithi083 ceph-mon[23711]: Reconfiguring crash.smithi083 (monmap changed)... 2024-09-18T07:20:48.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:48 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-18T07:20:48.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:48 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:48.662 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:48 smithi083 ceph-mon[23711]: Reconfiguring daemon crash.smithi083 on smithi083 2024-09-18T07:20:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:48 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:48 smithi096 ceph-mon[25928]: Reconfiguring crash.smithi083 (monmap changed)... 2024-09-18T07:20:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:48 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-18T07:20:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:48 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:48 smithi096 ceph-mon[25928]: Reconfiguring daemon crash.smithi083 on smithi083 2024-09-18T07:20:49.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:49 smithi083 ceph-mon[23711]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:49.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:49 smithi096 ceph-mon[25928]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:51.611 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:51 smithi083 ceph-mon[23711]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:51.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:51 smithi096 ceph-mon[25928]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:53.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:53 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:53.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:53 smithi083 ceph-mon[23711]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:53.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:53 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:53.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:53 smithi096 ceph-mon[25928]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:54.594 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:54 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:54.595 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:54 smithi083 ceph-mon[23711]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-09-18T07:20:54.595 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:54 smithi083 ceph-mon[23711]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-09-18T07:20:54.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:54 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:54.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:54 smithi096 ceph-mon[25928]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-09-18T07:20:54.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:54 smithi096 ceph-mon[25928]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-09-18T07:20:55.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:55 smithi096 ceph-mon[25928]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:55.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:55 smithi096 ceph-mon[25928]: from='osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-18T07:20:55.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:55 smithi083 ceph-mon[23711]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:55.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:55 smithi083 ceph-mon[23711]: from='osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-18T07:20:57.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:57 smithi083 ceph-mon[23711]: from='osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-18T07:20:57.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:57 smithi083 ceph-mon[23711]: osdmap e7: 1 total, 0 up, 1 in 2024-09-18T07:20:57.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:57 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:57.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:57 smithi083 ceph-mon[23711]: from='osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-09-18T07:20:57.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:57 smithi083 ceph-mon[23711]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:57.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:57 smithi096 ceph-mon[25928]: from='osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-18T07:20:57.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:57 smithi096 ceph-mon[25928]: osdmap e7: 1 total, 0 up, 1 in 2024-09-18T07:20:57.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:57 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:57.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:57 smithi096 ceph-mon[25928]: from='osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-09-18T07:20:57.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:57 smithi096 ceph-mon[25928]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:58.579 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:58 smithi096 ceph-mon[25928]: from='osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-09-18T07:20:58.579 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:58 smithi096 ceph-mon[25928]: osdmap e8: 1 total, 0 up, 1 in 2024-09-18T07:20:58.579 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:58.579 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:58.579 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:58.579 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi096", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-18T07:20:58.579 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:58 smithi083 ceph-mon[23711]: from='osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-09-18T07:20:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:58 smithi083 ceph-mon[23711]: osdmap e8: 1 total, 0 up, 1 in 2024-09-18T07:20:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:20:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi096", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-18T07:20:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:20:59.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:59 smithi083 ceph-mon[23711]: purged_snaps scrub starts 2024-09-18T07:20:59.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:59 smithi083 ceph-mon[23711]: purged_snaps scrub ok 2024-09-18T07:20:59.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:59 smithi083 ceph-mon[23711]: Reconfiguring crash.smithi096 (monmap changed)... 2024-09-18T07:20:59.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:59 smithi083 ceph-mon[23711]: Reconfiguring daemon crash.smithi096 on smithi096 2024-09-18T07:20:59.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:59 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:59.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:59 smithi083 ceph-mon[23711]: osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573] boot 2024-09-18T07:20:59.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:59 smithi083 ceph-mon[23711]: osdmap e9: 1 total, 1 up, 1 in 2024-09-18T07:20:59.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:59 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:59.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:20:59 smithi083 ceph-mon[23711]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:20:59.578 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 0 on host 'smithi083' 2024-09-18T07:20:59.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:59 smithi096 ceph-mon[25928]: purged_snaps scrub starts 2024-09-18T07:20:59.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:59 smithi096 ceph-mon[25928]: purged_snaps scrub ok 2024-09-18T07:20:59.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:59 smithi096 ceph-mon[25928]: Reconfiguring crash.smithi096 (monmap changed)... 2024-09-18T07:20:59.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:59 smithi096 ceph-mon[25928]: Reconfiguring daemon crash.smithi096 on smithi096 2024-09-18T07:20:59.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:59 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:59.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:59 smithi096 ceph-mon[25928]: osd.0 [v2:172.21.15.83:6802/3200079573,v1:172.21.15.83:6803/3200079573] boot 2024-09-18T07:20:59.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:59 smithi096 ceph-mon[25928]: osdmap e9: 1 total, 1 up, 1 in 2024-09-18T07:20:59.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:59 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:20:59.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:20:59 smithi096 ceph-mon[25928]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-18T07:21:00.252 DEBUG:teuthology.orchestra.run.smithi083:osd.0> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.0.service 2024-09-18T07:21:00.255 INFO:tasks.cephadm:Deploying osd.1 on smithi083 with /dev/vg_nvme/lv_3... 2024-09-18T07:21:00.256 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-18T07:21:00.596 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:00 smithi083 ceph-mon[23711]: osdmap e10: 1 total, 1 up, 1 in 2024-09-18T07:21:00.597 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:00.597 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:00 smithi083 ceph-mon[23711]: Reconfiguring mgr.smithi096.caifnv (monmap changed)... 2024-09-18T07:21:00.597 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:21:00.597 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:21:00.597 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:00.597 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:00 smithi083 ceph-mon[23711]: Reconfiguring daemon mgr.smithi096.caifnv on smithi096 2024-09-18T07:21:00.597 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:00.797 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:00 smithi096 ceph-mon[25928]: osdmap e10: 1 total, 1 up, 1 in 2024-09-18T07:21:00.797 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:00.797 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:00 smithi096 ceph-mon[25928]: Reconfiguring mgr.smithi096.caifnv (monmap changed)... 2024-09-18T07:21:00.797 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:21:00.798 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:21:00.798 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:00.798 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:00 smithi096 ceph-mon[25928]: Reconfiguring daemon mgr.smithi096.caifnv on smithi096 2024-09-18T07:21:00.798 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:01.712 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:01 smithi096 ceph-mon[25928]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:01.712 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:01.713 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-18T07:21:01.713 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-18T07:21:01.713 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:01.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:01 smithi083 ceph-mon[23711]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:01.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:01.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-18T07:21:01.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-18T07:21:01.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:02.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: Reconfiguring mon.smithi096 (monmap changed)... 2024-09-18T07:21:02.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: Reconfiguring daemon mon.smithi096 on smithi096 2024-09-18T07:21:02.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:02.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-18T07:21:02.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:02.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:02.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: Reconfiguring mon.smithi096 (monmap changed)... 2024-09-18T07:21:02.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: Reconfiguring daemon mon.smithi096 on smithi096 2024-09-18T07:21:02.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:02.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-18T07:21:02.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-09-18T07:21:02.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:02.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:21:02.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-09-18T07:21:02.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:02.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:21:02.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-09-18T07:21:02.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:02.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:02.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:02.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:02.880 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:21:02.900 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi083:vg_nvme/lv_3 2024-09-18T07:21:03.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:03 smithi096 ceph-mon[25928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-18T07:21:03.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:03 smithi096 ceph-mon[25928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-09-18T07:21:03.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:03 smithi096 ceph-mon[25928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:21:03.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:03 smithi096 ceph-mon[25928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-09-18T07:21:03.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:03 smithi096 ceph-mon[25928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:21:03.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:03 smithi096 ceph-mon[25928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-09-18T07:21:03.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:03 smithi096 ceph-mon[25928]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:03.766 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:03 smithi083 ceph-mon[23711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-18T07:21:03.767 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:03 smithi083 ceph-mon[23711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-09-18T07:21:03.767 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:03 smithi083 ceph-mon[23711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:21:03.767 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:03 smithi083 ceph-mon[23711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.83:3000"}]: dispatch 2024-09-18T07:21:03.767 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:03 smithi083 ceph-mon[23711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:21:03.767 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:03 smithi083 ceph-mon[23711]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.83:9095"}]: dispatch 2024-09-18T07:21:03.767 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:03 smithi083 ceph-mon[23711]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:04.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:04 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:04.932 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:04 smithi083 ceph-mon[23711]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:05.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:04 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:05.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:04 smithi096 ceph-mon[25928]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:06.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:06 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:21:06.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:06 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:21:06.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:06 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:06.660 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:06 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:21:06.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:06 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:21:06.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:06 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:07.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:07 smithi083 ceph-mon[23711]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:21:07.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:07 smithi083 ceph-mon[23711]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:07.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:07 smithi096 ceph-mon[25928]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:21:07.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:07 smithi096 ceph-mon[25928]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:09.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:09.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:09 smithi083 ceph-mon[23711]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:09.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:09.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:09 smithi096 ceph-mon[25928]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:10.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:10 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2123880217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e6e470d9-3890-49fa-81a4-e017a796053e"}]: dispatch 2024-09-18T07:21:10.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:10 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2123880217' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e6e470d9-3890-49fa-81a4-e017a796053e"}]': finished 2024-09-18T07:21:10.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:10 smithi083 ceph-mon[23711]: osdmap e11: 2 total, 1 up, 2 in 2024-09-18T07:21:10.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:10.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:10 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2123880217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e6e470d9-3890-49fa-81a4-e017a796053e"}]: dispatch 2024-09-18T07:21:10.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:10 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2123880217' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e6e470d9-3890-49fa-81a4-e017a796053e"}]': finished 2024-09-18T07:21:10.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:10 smithi096 ceph-mon[25928]: osdmap e11: 2 total, 1 up, 2 in 2024-09-18T07:21:10.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:10 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:11.617 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:11 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3077843326' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:21:11.617 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:11 smithi083 ceph-mon[23711]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:11.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:11 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/3077843326' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:21:11.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:11 smithi096 ceph-mon[25928]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:13.617 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:13 smithi083 ceph-mon[23711]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:13.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:13 smithi096 ceph-mon[25928]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:15 smithi096 ceph-mon[25928]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:21:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:15 smithi083 ceph-mon[23711]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:21:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:16.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:16 smithi083 ceph-mon[23711]: Detected new or changed devices on smithi083 2024-09-18T07:21:16.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:16 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi083 to 12864M 2024-09-18T07:21:16.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:16 smithi096 ceph-mon[25928]: Detected new or changed devices on smithi083 2024-09-18T07:21:16.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:16 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi083 to 12864M 2024-09-18T07:21:17.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:17 smithi083 ceph-mon[23711]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:17.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:17 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:17.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:17 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-18T07:21:17.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:17 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:17.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:17 smithi096 ceph-mon[25928]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:17 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:17 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-18T07:21:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:17 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:18.853 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:18 smithi083 ceph-mon[23711]: Deploying daemon osd.1 on smithi083 2024-09-18T07:21:18.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:18 smithi096 ceph-mon[25928]: Deploying daemon osd.1 on smithi083 2024-09-18T07:21:19.734 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:19 smithi083 ceph-mon[23711]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:19.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:19 smithi096 ceph-mon[25928]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:21.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:21 smithi083 ceph-mon[23711]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:21.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:21 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:21.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:21 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:21.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:21 smithi096 ceph-mon[25928]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:21 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:21 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:23.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:23 smithi083 ceph-mon[23711]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:23.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:23 smithi096 ceph-mon[25928]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:25.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:25 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:25.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:25 smithi096 ceph-mon[25928]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:25.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:25 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:25.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:25 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:25.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:25 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:25.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:25 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:25.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:25 smithi083 ceph-mon[23711]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:25.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:25 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:25.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:25 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:25.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:25 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:27.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:27.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:27 smithi083 ceph-mon[23711]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:27.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:27.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:27 smithi096 ceph-mon[25928]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:27.710 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 1 on host 'smithi083' 2024-09-18T07:21:28.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:28 smithi083 ceph-mon[23711]: from='osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-18T07:21:28.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:28 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:28.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:28 smithi096 ceph-mon[25928]: from='osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-18T07:21:28.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:28 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:28.659 DEBUG:teuthology.orchestra.run.smithi083:osd.1> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.1.service 2024-09-18T07:21:28.661 INFO:tasks.cephadm:Deploying osd.2 on smithi083 with /dev/vg_nvme/lv_2... 2024-09-18T07:21:28.661 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-18T07:21:29.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:29 smithi083 ceph-mon[23711]: from='osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-18T07:21:29.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:29 smithi083 ceph-mon[23711]: osdmap e12: 2 total, 1 up, 2 in 2024-09-18T07:21:29.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:29.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:29 smithi083 ceph-mon[23711]: from='osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-09-18T07:21:29.507 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:29 smithi083 ceph-mon[23711]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:29.508 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:21:29 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[38155]: 2024-09-18T07:21:29.198+0000 7fe27a396700 -1 osd.1 0 waiting for initial osdmap 2024-09-18T07:21:29.508 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:21:29 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[38155]: 2024-09-18T07:21:29.203+0000 7fe272528700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:21:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:29 smithi096 ceph-mon[25928]: from='osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-18T07:21:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:29 smithi096 ceph-mon[25928]: osdmap e12: 2 total, 1 up, 2 in 2024-09-18T07:21:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:29 smithi096 ceph-mon[25928]: from='osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-09-18T07:21:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:29 smithi096 ceph-mon[25928]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:30.510 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:30 smithi083 ceph-mon[23711]: from='osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-09-18T07:21:30.511 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:30 smithi083 ceph-mon[23711]: osdmap e13: 2 total, 1 up, 2 in 2024-09-18T07:21:30.511 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:30 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:30.511 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:30 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:30.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:30 smithi096 ceph-mon[25928]: from='osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-09-18T07:21:30.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:30 smithi096 ceph-mon[25928]: osdmap e13: 2 total, 1 up, 2 in 2024-09-18T07:21:30.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:30 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:30.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:30 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:31.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:31 smithi083 ceph-mon[23711]: purged_snaps scrub starts 2024-09-18T07:21:31.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:31 smithi083 ceph-mon[23711]: purged_snaps scrub ok 2024-09-18T07:21:31.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:31 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:31.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:31 smithi083 ceph-mon[23711]: osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401] boot 2024-09-18T07:21:31.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:31 smithi083 ceph-mon[23711]: osdmap e14: 2 total, 2 up, 2 in 2024-09-18T07:21:31.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:31 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:31.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:31 smithi083 ceph-mon[23711]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:31.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:31 smithi096 ceph-mon[25928]: purged_snaps scrub starts 2024-09-18T07:21:31.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:31 smithi096 ceph-mon[25928]: purged_snaps scrub ok 2024-09-18T07:21:31.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:31 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:31.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:31 smithi096 ceph-mon[25928]: osd.1 [v2:172.21.15.83:6810/24696401,v1:172.21.15.83:6811/24696401] boot 2024-09-18T07:21:31.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:31 smithi096 ceph-mon[25928]: osdmap e14: 2 total, 2 up, 2 in 2024-09-18T07:21:31.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:31 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:21:31.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:31 smithi096 ceph-mon[25928]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-18T07:21:31.793 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:21:31.814 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi083:vg_nvme/lv_2 2024-09-18T07:21:32.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:32 smithi083 ceph-mon[23711]: osdmap e15: 2 total, 2 up, 2 in 2024-09-18T07:21:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:32 smithi096 ceph-mon[25928]: osdmap e15: 2 total, 2 up, 2 in 2024-09-18T07:21:33.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:33 smithi083 ceph-mon[23711]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:33.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:33 smithi096 ceph-mon[25928]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:34.595 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:34 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:21:34.595 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:34 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:21:34.596 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:34 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:34.596 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:34 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:34 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:21:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:34 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:21:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:34 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:34 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:35.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:35 smithi096 ceph-mon[25928]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:21:35.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:35 smithi096 ceph-mon[25928]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:35.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:35 smithi083 ceph-mon[23711]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:21:35.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:35 smithi083 ceph-mon[23711]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:37 smithi083 ceph-mon[23711]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:37 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1232916678' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b104b18e-f2d0-4a92-86d6-068c62835a6a"}]: dispatch 2024-09-18T07:21:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:37 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1232916678' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b104b18e-f2d0-4a92-86d6-068c62835a6a"}]': finished 2024-09-18T07:21:37.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:37 smithi083 ceph-mon[23711]: osdmap e16: 3 total, 2 up, 3 in 2024-09-18T07:21:37.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:37 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:37.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:37 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1668938859' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:21:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:37 smithi096 ceph-mon[25928]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:37 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1232916678' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b104b18e-f2d0-4a92-86d6-068c62835a6a"}]: dispatch 2024-09-18T07:21:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:37 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1232916678' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b104b18e-f2d0-4a92-86d6-068c62835a6a"}]': finished 2024-09-18T07:21:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:37 smithi096 ceph-mon[25928]: osdmap e16: 3 total, 2 up, 3 in 2024-09-18T07:21:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:37 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:37 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1668938859' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:21:39.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:39 smithi083 ceph-mon[23711]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:39.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:39 smithi096 ceph-mon[25928]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:41.611 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:41 smithi083 ceph-mon[23711]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:41.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:41 smithi096 ceph-mon[25928]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:43.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: Detected new or changed devices on smithi083 2024-09-18T07:21:43.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:43.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:21:43.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:21:43.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi083 to 6432M 2024-09-18T07:21:43.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:43.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:43.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:43.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:43.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:43.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:43 smithi083 ceph-mon[23711]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:43.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: Detected new or changed devices on smithi083 2024-09-18T07:21:43.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:43.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:21:43.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:21:43.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi083 to 6432M 2024-09-18T07:21:43.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:43.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:43.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:43.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:43.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:43.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:43 smithi096 ceph-mon[25928]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:44.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-18T07:21:44.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:44.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-18T07:21:44.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:45.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:45 smithi083 ceph-mon[23711]: Deploying daemon osd.2 on smithi083 2024-09-18T07:21:45.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:45 smithi083 ceph-mon[23711]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:45.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:21:45.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:21:45.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:45 smithi096 ceph-mon[25928]: Deploying daemon osd.2 on smithi083 2024-09-18T07:21:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:45 smithi096 ceph-mon[25928]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:45 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:21:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:45 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:21:47.794 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:47 smithi083 ceph-mon[23711]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:47.869 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:47 smithi096 ceph-mon[25928]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:49.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:49 smithi083 ceph-mon[23711]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:49.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:49 smithi096 ceph-mon[25928]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:51.013 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:50 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:51.014 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:50 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:51.014 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:50 smithi083 ceph-mon[23711]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:51.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:50 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:51.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:50 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:51.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:50 smithi096 ceph-mon[25928]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:52.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:52 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:52.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:52 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:52.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:52 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:52.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:52 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:52.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:52 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:52.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:52 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:21:52.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:52 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:21:52.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:52 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:21:53.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:53 smithi083 ceph-mon[23711]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:53.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:53 smithi096 ceph-mon[25928]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:54.025 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 2 on host 'smithi083' 2024-09-18T07:21:54.636 DEBUG:teuthology.orchestra.run.smithi083:osd.2> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.2.service 2024-09-18T07:21:54.639 INFO:tasks.cephadm:Deploying osd.3 on smithi083 with /dev/vg_nvme/lv_1... 2024-09-18T07:21:54.639 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-18T07:21:55.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:55 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:55.344 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:55 smithi083 ceph-mon[23711]: from='osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-18T07:21:55.345 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:55 smithi083 ceph-mon[23711]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:55.356 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:55 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:21:55.356 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:55 smithi096 ceph-mon[25928]: from='osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-18T07:21:55.356 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:55 smithi096 ceph-mon[25928]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:56.206 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:56 smithi083 ceph-mon[23711]: from='osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-18T07:21:56.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:56 smithi083 ceph-mon[23711]: osdmap e17: 3 total, 2 up, 3 in 2024-09-18T07:21:56.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:56 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:56.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:56 smithi083 ceph-mon[23711]: from='osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-09-18T07:21:56.207 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:21:56 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[41804]: 2024-09-18T07:21:56.031+0000 7fe3c8201700 -1 osd.2 0 waiting for initial osdmap 2024-09-18T07:21:56.207 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:21:56 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[41804]: 2024-09-18T07:21:56.035+0000 7fe3c0b94700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:21:56.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:56 smithi096 ceph-mon[25928]: from='osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-18T07:21:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:56 smithi096 ceph-mon[25928]: osdmap e17: 3 total, 2 up, 3 in 2024-09-18T07:21:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:56 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:56 smithi096 ceph-mon[25928]: from='osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-09-18T07:21:57.364 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:57 smithi083 ceph-mon[23711]: from='osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-09-18T07:21:57.384 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:57 smithi083 ceph-mon[23711]: osdmap e18: 3 total, 2 up, 3 in 2024-09-18T07:21:57.384 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:57 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:57.384 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:57 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:57.384 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:57 smithi083 ceph-mon[23711]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:57.384 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:57 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:57.385 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:57 smithi096 ceph-mon[25928]: from='osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-09-18T07:21:57.385 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:57 smithi096 ceph-mon[25928]: osdmap e18: 3 total, 2 up, 3 in 2024-09-18T07:21:57.385 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:57 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:57.386 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:57 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:57.386 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:57 smithi096 ceph-mon[25928]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-18T07:21:57.386 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:57 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:57.386 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:21:57.388 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi083:vg_nvme/lv_1 2024-09-18T07:21:58.153 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:58 smithi083 ceph-mon[23711]: purged_snaps scrub starts 2024-09-18T07:21:58.153 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:58 smithi083 ceph-mon[23711]: purged_snaps scrub ok 2024-09-18T07:21:58.153 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:58 smithi083 ceph-mon[23711]: osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812] boot 2024-09-18T07:21:58.153 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:58 smithi083 ceph-mon[23711]: osdmap e19: 3 total, 3 up, 3 in 2024-09-18T07:21:58.153 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:58.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:58 smithi096 ceph-mon[25928]: purged_snaps scrub starts 2024-09-18T07:21:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:58 smithi096 ceph-mon[25928]: purged_snaps scrub ok 2024-09-18T07:21:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:58 smithi096 ceph-mon[25928]: osd.2 [v2:172.21.15.83:6818/1158890812,v1:172.21.15.83:6819/1158890812] boot 2024-09-18T07:21:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:58 smithi096 ceph-mon[25928]: osdmap e19: 3 total, 3 up, 3 in 2024-09-18T07:21:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:21:59.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:59 smithi083 ceph-mon[23711]: osdmap e20: 3 total, 3 up, 3 in 2024-09-18T07:21:59.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:59 smithi083 ceph-mon[23711]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:21:59.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:21:59 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-18T07:21:59.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:59 smithi096 ceph-mon[25928]: osdmap e20: 3 total, 3 up, 3 in 2024-09-18T07:21:59.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:59 smithi096 ceph-mon[25928]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:21:59.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:21:59 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-18T07:22:00.176 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-18T07:22:00.176 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:00 smithi083 ceph-mon[23711]: osdmap e21: 3 total, 3 up, 3 in 2024-09-18T07:22:00.176 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-18T07:22:00.176 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:00 smithi083 ceph-mon[23711]: from='client.14288 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:22:00.176 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:22:00.176 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:22:00.176 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:00 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:00.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-18T07:22:00.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:00 smithi096 ceph-mon[25928]: osdmap e21: 3 total, 3 up, 3 in 2024-09-18T07:22:00.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-18T07:22:00.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:00 smithi096 ceph-mon[25928]: from='client.14288 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:22:00.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:22:00.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:22:00.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:00 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:00.893 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:22:00 smithi083 sudo[43200]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-18T07:22:00.893 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:22:00 smithi083 sudo[43200]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:00.893 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:22:00 smithi083 sudo[43200]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:00.893 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:22:00 smithi083 sudo[43223]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-18T07:22:01.166 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:22:00 smithi083 sudo[43223]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:01.166 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:22:00 smithi083 sudo[43223]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:01.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-18T07:22:01.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:01 smithi083 ceph-mon[23711]: osdmap e22: 3 total, 3 up, 3 in 2024-09-18T07:22:01.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:01 smithi083 ceph-mon[23711]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:01.167 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:01.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-18T07:22:01.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:01 smithi096 ceph-mon[25928]: osdmap e22: 3 total, 3 up, 3 in 2024-09-18T07:22:01.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:01 smithi096 ceph-mon[25928]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:01.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:01.426 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43228]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-18T07:22:01.427 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43228]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:01.427 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43228]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:01.681 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43249]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-18T07:22:01.681 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43249]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:01.681 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43249]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:01.942 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43299]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-18T07:22:01.942 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43299]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:01.942 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43299]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:02.197 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43328]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-18T07:22:02.197 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43328]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:02.198 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:22:01 smithi083 sudo[43328]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:02.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:02 smithi083 sudo[43336]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-18T07:22:02.198 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:02 smithi083 sudo[43336]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:02 smithi083 sudo[43336]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:02 smithi083 sudo[43339]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-18T07:22:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:02 smithi083 sudo[43339]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:02.447 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:02 smithi083 sudo[43339]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:02.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:02 smithi083 ceph-mon[23711]: osdmap e23: 3 total, 3 up, 3 in 2024-09-18T07:22:02.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:02 smithi083 ceph-mon[23711]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-18T07:22:02.708 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:02 smithi096 ceph-mon[25928]: osdmap e23: 3 total, 3 up, 3 in 2024-09-18T07:22:02.708 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:02 smithi096 ceph-mon[25928]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-18T07:22:02.708 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:02 smithi096 sudo[28161]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-18T07:22:02.708 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:02 smithi096 sudo[28161]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:02.708 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:02 smithi096 sudo[28161]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:03.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:02 smithi096 sudo[28164]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-18T07:22:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:02 smithi096 sudo[28164]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-18T07:22:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:02 smithi096 sudo[28164]: pam_unix(sudo:session): session closed for user root 2024-09-18T07:22:03.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-18T07:22:03.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:22:03.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:22:03.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-18T07:22:03.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: mgrmap e19: smithi083.vchvnu(active, since 2m), standbys: smithi096.caifnv 2024-09-18T07:22:03.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:22:03.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:22:03.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:03.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-18T07:22:03.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:03.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:03.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/115862598' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48c6e315-85de-413e-b9a2-8a067af7a090"}]: dispatch 2024-09-18T07:22:03.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/115862598' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "48c6e315-85de-413e-b9a2-8a067af7a090"}]': finished 2024-09-18T07:22:03.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: osdmap e24: 4 total, 3 up, 4 in 2024-09-18T07:22:03.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:03.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-18T07:22:03.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:22:03.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:22:03.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-18T07:22:03.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: mgrmap e19: smithi083.vchvnu(active, since 2m), standbys: smithi096.caifnv 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/115862598' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "48c6e315-85de-413e-b9a2-8a067af7a090"}]: dispatch 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/115862598' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "48c6e315-85de-413e-b9a2-8a067af7a090"}]': finished 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: osdmap e24: 4 total, 3 up, 4 in 2024-09-18T07:22:03.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:04 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2368854323' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:22:04.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:04 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2368854323' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:22:05.523 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:05 smithi096 ceph-mon[25928]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:05.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:05 smithi083 ceph-mon[23711]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:07.799 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:07 smithi083 ceph-mon[23711]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-18T07:22:07.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:07 smithi096 ceph-mon[25928]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-18T07:22:09.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-18T07:22:09.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:09.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:09.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:09.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:09.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:09.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:09.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:09.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:09.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:09.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:09 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:09.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:09 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:10.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:10 smithi083 ceph-mon[23711]: Detected new or changed devices on smithi083 2024-09-18T07:22:10.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:10 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi083 to 4288M 2024-09-18T07:22:10.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-18T07:22:10.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:10 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:10.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:10 smithi096 ceph-mon[25928]: Detected new or changed devices on smithi083 2024-09-18T07:22:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:10 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi083 to 4288M 2024-09-18T07:22:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:10 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-18T07:22:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:10 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:11.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:11 smithi083 ceph-mon[23711]: Deploying daemon osd.3 on smithi083 2024-09-18T07:22:11.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:11 smithi083 ceph-mon[23711]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-18T07:22:11.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:11 smithi096 ceph-mon[25928]: Deploying daemon osd.3 on smithi083 2024-09-18T07:22:11.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:11 smithi096 ceph-mon[25928]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-18T07:22:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:13 smithi083 ceph-mon[23711]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-18T07:22:13.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:13 smithi096 ceph-mon[25928]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-18T07:22:15.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:15 smithi096 ceph-mon[25928]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 4 op/s 2024-09-18T07:22:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:15 smithi083 ceph-mon[23711]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 4 op/s 2024-09-18T07:22:16.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:16 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:16.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:16 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:16.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:16 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:16.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:16 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:16 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:16 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:16 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:16 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:17.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:17 smithi083 ceph-mon[23711]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-18T07:22:18.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:17 smithi096 ceph-mon[25928]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-18T07:22:19.492 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 3 on host 'smithi083' 2024-09-18T07:22:19.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:19 smithi083 ceph-mon[23711]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:19.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:19 smithi083 ceph-mon[23711]: from='osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-18T07:22:19.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:19 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:20.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:19 smithi096 ceph-mon[25928]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:19 smithi096 ceph-mon[25928]: from='osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-18T07:22:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:19 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:20.182 DEBUG:teuthology.orchestra.run.smithi083:osd.3> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.3.service 2024-09-18T07:22:20.185 INFO:tasks.cephadm:Deploying osd.4 on smithi096 with /dev/vg_nvme/lv_4... 2024-09-18T07:22:20.185 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-18T07:22:20.823 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:20 smithi096 ceph-mon[25928]: from='osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-18T07:22:20.823 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:20 smithi096 ceph-mon[25928]: osdmap e25: 4 total, 3 up, 4 in 2024-09-18T07:22:20.823 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:20 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:20.823 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:20 smithi096 ceph-mon[25928]: from='osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-09-18T07:22:20.823 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:20 smithi096 ceph-mon[25928]: from='osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-09-18T07:22:20.824 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:20 smithi096 ceph-mon[25928]: osdmap e26: 4 total, 3 up, 4 in 2024-09-18T07:22:20.824 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:20 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:20.824 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:20 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:20.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-mon[23711]: from='osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-18T07:22:20.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-mon[23711]: osdmap e25: 4 total, 3 up, 4 in 2024-09-18T07:22:20.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:20.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-mon[23711]: from='osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-09-18T07:22:20.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-mon[23711]: from='osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-09-18T07:22:20.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-mon[23711]: osdmap e26: 4 total, 3 up, 4 in 2024-09-18T07:22:20.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:20.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:20.869 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[45110]: 2024-09-18T07:22:20.695+0000 7f7f15c38700 -1 osd.3 0 waiting for initial osdmap 2024-09-18T07:22:20.869 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:22:20 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[45110]: 2024-09-18T07:22:20.701+0000 7f7f0f5cd700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:22:21.963 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:21 smithi083 ceph-mon[23711]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:21.963 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:21 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:22.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:21 smithi096 ceph-mon[25928]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:22.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:21 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:22.762 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:22:22.780 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:22 smithi096 ceph-mon[25928]: purged_snaps scrub starts 2024-09-18T07:22:22.780 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:22 smithi096 ceph-mon[25928]: purged_snaps scrub ok 2024-09-18T07:22:22.780 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:22 smithi096 ceph-mon[25928]: osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851] boot 2024-09-18T07:22:22.780 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:22 smithi096 ceph-mon[25928]: osdmap e27: 4 total, 4 up, 4 in 2024-09-18T07:22:22.781 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:22 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:22.781 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:22 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:22.782 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi096:vg_nvme/lv_4 2024-09-18T07:22:23.019 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:22 smithi083 ceph-mon[23711]: purged_snaps scrub starts 2024-09-18T07:22:23.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:22 smithi083 ceph-mon[23711]: purged_snaps scrub ok 2024-09-18T07:22:23.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:22 smithi083 ceph-mon[23711]: osd.3 [v2:172.21.15.83:6826/4086177851,v1:172.21.15.83:6827/4086177851] boot 2024-09-18T07:22:23.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:22 smithi083 ceph-mon[23711]: osdmap e27: 4 total, 4 up, 4 in 2024-09-18T07:22:23.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:22 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:22:23.020 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:22 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:23.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:23 smithi096 ceph-mon[25928]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:23.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:23 smithi096 ceph-mon[25928]: osdmap e28: 4 total, 4 up, 4 in 2024-09-18T07:22:24.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:23 smithi083 ceph-mon[23711]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-18T07:22:24.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:23 smithi083 ceph-mon[23711]: osdmap e28: 4 total, 4 up, 4 in 2024-09-18T07:22:24.834 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:24 smithi096 ceph-mon[25928]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:22:24.834 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:24 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:22:24.834 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:24 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:22:24.834 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:24 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:24.835 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:24 smithi096 ceph-mon[25928]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:25.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:24 smithi083 ceph-mon[23711]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:22:25.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:24 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:22:25.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:24 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:22:25.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:24 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:25.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:24 smithi083 ceph-mon[23711]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi083 to 3216M 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:26.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:26.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:26.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:26 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi083 to 3216M 2024-09-18T07:22:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:26.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:26.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:26.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:26 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:27.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:27 smithi083 ceph-mon[23711]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:27.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:27 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/812764524' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0cef4e7-db42-4b46-a43e-dc596efcc266"}]: dispatch 2024-09-18T07:22:27.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:27 smithi083 ceph-mon[23711]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0cef4e7-db42-4b46-a43e-dc596efcc266"}]: dispatch 2024-09-18T07:22:27.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:27 smithi083 ceph-mon[23711]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e0cef4e7-db42-4b46-a43e-dc596efcc266"}]': finished 2024-09-18T07:22:27.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:27 smithi083 ceph-mon[23711]: osdmap e29: 5 total, 4 up, 5 in 2024-09-18T07:22:27.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:27.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:27 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/1526562004' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:22:27.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:27 smithi096 ceph-mon[25928]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:27.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:27 smithi096 ceph-mon[25928]: from='client.? 172.21.15.96:0/812764524' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0cef4e7-db42-4b46-a43e-dc596efcc266"}]: dispatch 2024-09-18T07:22:27.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:27 smithi096 ceph-mon[25928]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0cef4e7-db42-4b46-a43e-dc596efcc266"}]: dispatch 2024-09-18T07:22:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:27 smithi096 ceph-mon[25928]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e0cef4e7-db42-4b46-a43e-dc596efcc266"}]': finished 2024-09-18T07:22:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:27 smithi096 ceph-mon[25928]: osdmap e29: 5 total, 4 up, 5 in 2024-09-18T07:22:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:27 smithi096 ceph-mon[25928]: from='client.? 172.21.15.96:0/1526562004' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:22:29.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:29 smithi096 ceph-mon[25928]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:29.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:29 smithi083 ceph-mon[23711]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:31.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:31 smithi096 ceph-mon[25928]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:31.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:31 smithi083 ceph-mon[23711]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:32.823 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:32 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-18T07:22:32.823 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:32 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:32.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:32 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-18T07:22:32.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:32 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:33.864 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:33 smithi096 ceph-mon[25928]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:33.864 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:33 smithi096 ceph-mon[25928]: Deploying daemon osd.4 on smithi096 2024-09-18T07:22:33.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:33 smithi083 ceph-mon[23711]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:33.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:33 smithi083 ceph-mon[23711]: Deploying daemon osd.4 on smithi096 2024-09-18T07:22:35.857 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:35 smithi096 ceph-mon[25928]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:35.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:35 smithi083 ceph-mon[23711]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:37.820 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:37 smithi096 ceph-mon[25928]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:37.821 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:37 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:37.821 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:37 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:37.821 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:37 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:37.821 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:37 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:37 smithi083 ceph-mon[23711]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:37 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:37 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:37 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:37 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:39.854 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:39 smithi096 ceph-mon[25928]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:39.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:39 smithi083 ceph-mon[23711]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:40.458 INFO:teuthology.orchestra.run.smithi096.stdout:Created osd(s) 4 on host 'smithi096' 2024-09-18T07:22:41.164 DEBUG:teuthology.orchestra.run.smithi096:osd.4> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.4.service 2024-09-18T07:22:41.167 INFO:tasks.cephadm:Deploying osd.5 on smithi096 with /dev/vg_nvme/lv_3... 2024-09-18T07:22:41.167 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-18T07:22:41.840 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:41 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:41.841 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:41 smithi096 ceph-mon[25928]: from='osd.4 [v2:172.21.15.96:6800/2765851169,v1:172.21.15.96:6801/2765851169]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-18T07:22:41.841 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:41 smithi096 ceph-mon[25928]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-18T07:22:41.841 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:41 smithi096 ceph-mon[25928]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:41.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:41 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:41.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:41 smithi083 ceph-mon[23711]: from='osd.4 [v2:172.21.15.96:6800/2765851169,v1:172.21.15.96:6801/2765851169]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-18T07:22:41.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:41 smithi083 ceph-mon[23711]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-18T07:22:41.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:41 smithi083 ceph-mon[23711]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:42.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:42 smithi096 ceph-mon[25928]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-18T07:22:42.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:42 smithi096 ceph-mon[25928]: osdmap e30: 5 total, 4 up, 5 in 2024-09-18T07:22:42.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:42 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:42.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:42 smithi096 ceph-mon[25928]: from='osd.4 [v2:172.21.15.96:6800/2765851169,v1:172.21.15.96:6801/2765851169]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:22:42.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:42 smithi096 ceph-mon[25928]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:22:42.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:42 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:42.786 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:22:42 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[30068]: 2024-09-18T07:22:42.466+0000 7f43e7eca700 -1 osd.4 0 waiting for initial osdmap 2024-09-18T07:22:42.786 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:22:42 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[30068]: 2024-09-18T07:22:42.470+0000 7f43e2861700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:22:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:42 smithi083 ceph-mon[23711]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-18T07:22:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:42 smithi083 ceph-mon[23711]: osdmap e30: 5 total, 4 up, 5 in 2024-09-18T07:22:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:42 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:42 smithi083 ceph-mon[23711]: from='osd.4 [v2:172.21.15.96:6800/2765851169,v1:172.21.15.96:6801/2765851169]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:22:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:42 smithi083 ceph-mon[23711]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:22:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:42 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:43.798 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:43 smithi096 ceph-mon[25928]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-18T07:22:43.798 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:43 smithi096 ceph-mon[25928]: osdmap e31: 5 total, 4 up, 5 in 2024-09-18T07:22:43.799 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:43.799 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:43 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:43.799 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:43 smithi096 ceph-mon[25928]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:43.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:43 smithi083 ceph-mon[23711]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-18T07:22:43.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:43 smithi083 ceph-mon[23711]: osdmap e31: 5 total, 4 up, 5 in 2024-09-18T07:22:43.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:43.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:43 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:43.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:43 smithi083 ceph-mon[23711]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-18T07:22:44.854 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:44 smithi096 ceph-mon[25928]: purged_snaps scrub starts 2024-09-18T07:22:44.855 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:44 smithi096 ceph-mon[25928]: purged_snaps scrub ok 2024-09-18T07:22:44.855 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:44 smithi096 ceph-mon[25928]: osd.4 [v2:172.21.15.96:6800/2765851169,v1:172.21.15.96:6801/2765851169] boot 2024-09-18T07:22:44.855 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:44 smithi096 ceph-mon[25928]: osdmap e32: 5 total, 5 up, 5 in 2024-09-18T07:22:44.855 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:44.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:44 smithi083 ceph-mon[23711]: purged_snaps scrub starts 2024-09-18T07:22:44.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:44 smithi083 ceph-mon[23711]: purged_snaps scrub ok 2024-09-18T07:22:44.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:44 smithi083 ceph-mon[23711]: osd.4 [v2:172.21.15.96:6800/2765851169,v1:172.21.15.96:6801/2765851169] boot 2024-09-18T07:22:44.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:44 smithi083 ceph-mon[23711]: osdmap e32: 5 total, 5 up, 5 in 2024-09-18T07:22:44.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:22:45.798 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:45 smithi096 ceph-mon[25928]: osdmap e33: 5 total, 5 up, 5 in 2024-09-18T07:22:45.798 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:45 smithi096 ceph-mon[25928]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:22:45.798 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:45 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:22:45.799 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:45 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:22:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:45 smithi083 ceph-mon[23711]: osdmap e33: 5 total, 5 up, 5 in 2024-09-18T07:22:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:45 smithi083 ceph-mon[23711]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:22:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:22:45.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:22:46.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: osdmap e34: 5 total, 5 up, 5 in 2024-09-18T07:22:46.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: Detected new or changed devices on smithi096 2024-09-18T07:22:46.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:46.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:46.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi096 to 15936M 2024-09-18T07:22:46.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:46.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:46.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:46.730 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:46.730 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:46.730 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:46.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: osdmap e34: 5 total, 5 up, 5 in 2024-09-18T07:22:46.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: Detected new or changed devices on smithi096 2024-09-18T07:22:46.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:46.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:22:46.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi096 to 15936M 2024-09-18T07:22:46.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:46.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:46.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:22:46.870 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:46.870 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:22:46.870 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:22:47.051 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:22:47.071 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi096:vg_nvme/lv_3 2024-09-18T07:22:47.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:47 smithi096 ceph-mon[25928]: osdmap e35: 5 total, 5 up, 5 in 2024-09-18T07:22:47.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:47 smithi096 ceph-mon[25928]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-18T07:22:47.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:47 smithi083 ceph-mon[23711]: osdmap e35: 5 total, 5 up, 5 in 2024-09-18T07:22:47.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:47 smithi083 ceph-mon[23711]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-18T07:22:48.536 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:48 smithi096 ceph-mon[25928]: osdmap e36: 5 total, 5 up, 5 in 2024-09-18T07:22:48.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:48 smithi083 ceph-mon[23711]: osdmap e36: 5 total, 5 up, 5 in 2024-09-18T07:22:49.550 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:49 smithi096 ceph-mon[25928]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-09-18T07:22:49.550 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:22:49.550 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:22:49.550 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:49.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:49 smithi083 ceph-mon[23711]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 128 KiB/s, 0 objects/s recovering 2024-09-18T07:22:49.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:22:49.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:22:49.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:50.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:50 smithi083 ceph-mon[23711]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:22:50.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:50 smithi096 ceph-mon[25928]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:22:51.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:51 smithi083 ceph-mon[23711]: pgmap v117: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-18T07:22:51.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:51 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/2435996346' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc03009-1d9f-4b1b-84ec-0be59727452c"}]: dispatch 2024-09-18T07:22:51.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:51 smithi083 ceph-mon[23711]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc03009-1d9f-4b1b-84ec-0be59727452c"}]: dispatch 2024-09-18T07:22:51.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:51 smithi083 ceph-mon[23711]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dcc03009-1d9f-4b1b-84ec-0be59727452c"}]': finished 2024-09-18T07:22:51.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:51 smithi083 ceph-mon[23711]: osdmap e37: 6 total, 5 up, 6 in 2024-09-18T07:22:51.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:22:51.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:51 smithi096 ceph-mon[25928]: pgmap v117: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-18T07:22:51.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:51 smithi096 ceph-mon[25928]: from='client.? 172.21.15.96:0/2435996346' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc03009-1d9f-4b1b-84ec-0be59727452c"}]: dispatch 2024-09-18T07:22:51.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:51 smithi096 ceph-mon[25928]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcc03009-1d9f-4b1b-84ec-0be59727452c"}]: dispatch 2024-09-18T07:22:51.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:51 smithi096 ceph-mon[25928]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dcc03009-1d9f-4b1b-84ec-0be59727452c"}]': finished 2024-09-18T07:22:51.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:51 smithi096 ceph-mon[25928]: osdmap e37: 6 total, 5 up, 6 in 2024-09-18T07:22:51.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:51 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:22:52.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:52 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/1504773553' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:22:52.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:52 smithi096 ceph-mon[25928]: from='client.? 172.21.15.96:0/1504773553' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:22:53.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:53 smithi083 ceph-mon[23711]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:22:53.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:53 smithi096 ceph-mon[25928]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:22:55.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:55 smithi096 ceph-mon[25928]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:22:55.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:55 smithi083 ceph-mon[23711]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:22:57.653 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:57 smithi096 ceph-mon[25928]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:22:57.653 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:57 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-18T07:22:57.653 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:57 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:57 smithi083 ceph-mon[23711]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:22:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:57 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-18T07:22:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:57 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:22:58.837 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:58 smithi096 ceph-mon[25928]: Deploying daemon osd.5 on smithi096 2024-09-18T07:22:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:58 smithi083 ceph-mon[23711]: Deploying daemon osd.5 on smithi096 2024-09-18T07:22:59.835 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:22:59 smithi096 ceph-mon[25928]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:22:59.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:22:59 smithi083 ceph-mon[23711]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:01.760 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:01 smithi096 ceph-mon[25928]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:01.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:01 smithi083 ceph-mon[23711]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:03.769 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:03.769 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:03.769 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:03.769 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:03 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:03.769 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:03 smithi096 ceph-mon[25928]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:03.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:03.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:03.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:03.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:03 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:03.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:03 smithi083 ceph-mon[23711]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:05.421 INFO:teuthology.orchestra.run.smithi096.stdout:Created osd(s) 5 on host 'smithi096' 2024-09-18T07:23:05.722 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:05 smithi096 ceph-mon[25928]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:05.722 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:05 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:05.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:05 smithi083 ceph-mon[23711]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:05.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:05 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:06.021 DEBUG:teuthology.orchestra.run.smithi096:osd.5> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.5.service 2024-09-18T07:23:06.024 INFO:tasks.cephadm:Deploying osd.6 on smithi096 with /dev/vg_nvme/lv_2... 2024-09-18T07:23:06.024 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-18T07:23:06.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:06 smithi083 ceph-mon[23711]: from='osd.5 [v2:172.21.15.96:6808/692248156,v1:172.21.15.96:6809/692248156]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-18T07:23:06.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:06 smithi083 ceph-mon[23711]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-18T07:23:06.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:06 smithi096 ceph-mon[25928]: from='osd.5 [v2:172.21.15.96:6808/692248156,v1:172.21.15.96:6809/692248156]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-18T07:23:06.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:06 smithi096 ceph-mon[25928]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-18T07:23:07.841 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:07 smithi096 ceph-mon[25928]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:07.841 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:07 smithi096 ceph-mon[25928]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-18T07:23:07.841 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:07 smithi096 ceph-mon[25928]: osdmap e38: 6 total, 5 up, 6 in 2024-09-18T07:23:07.841 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:07 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:23:07.842 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:07 smithi096 ceph-mon[25928]: from='osd.5 [v2:172.21.15.96:6808/692248156,v1:172.21.15.96:6809/692248156]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:07.842 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:07 smithi096 ceph-mon[25928]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:07.842 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:07 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:07.842 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:23:07 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[33022]: 2024-09-18T07:23:07.593+0000 7ff169c15700 -1 osd.5 0 waiting for initial osdmap 2024-09-18T07:23:07.842 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:23:07 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[33022]: 2024-09-18T07:23:07.599+0000 7ff164dad700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:23:07.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:07 smithi083 ceph-mon[23711]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:07.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:07 smithi083 ceph-mon[23711]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-18T07:23:07.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:07 smithi083 ceph-mon[23711]: osdmap e38: 6 total, 5 up, 6 in 2024-09-18T07:23:07.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:07 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:23:07.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:07 smithi083 ceph-mon[23711]: from='osd.5 [v2:172.21.15.96:6808/692248156,v1:172.21.15.96:6809/692248156]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:07.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:07 smithi083 ceph-mon[23711]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:07.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:07 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:08.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:08 smithi096 ceph-mon[25928]: purged_snaps scrub starts 2024-09-18T07:23:08.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:08 smithi096 ceph-mon[25928]: purged_snaps scrub ok 2024-09-18T07:23:08.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:08 smithi096 ceph-mon[25928]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-18T07:23:08.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:08 smithi096 ceph-mon[25928]: osdmap e39: 6 total, 5 up, 6 in 2024-09-18T07:23:08.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:08 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:23:08.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:08 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:23:08.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:08 smithi096 ceph-mon[25928]: osd.5 [v2:172.21.15.96:6808/692248156,v1:172.21.15.96:6809/692248156] boot 2024-09-18T07:23:08.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:08 smithi096 ceph-mon[25928]: osdmap e40: 6 total, 6 up, 6 in 2024-09-18T07:23:08.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:08 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:23:08.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:08 smithi083 ceph-mon[23711]: purged_snaps scrub starts 2024-09-18T07:23:08.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:08 smithi083 ceph-mon[23711]: purged_snaps scrub ok 2024-09-18T07:23:08.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:08 smithi083 ceph-mon[23711]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-18T07:23:08.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:08 smithi083 ceph-mon[23711]: osdmap e39: 6 total, 5 up, 6 in 2024-09-18T07:23:08.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:08 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:23:08.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:08 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:23:08.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:08 smithi083 ceph-mon[23711]: osd.5 [v2:172.21.15.96:6808/692248156,v1:172.21.15.96:6809/692248156] boot 2024-09-18T07:23:08.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:08 smithi083 ceph-mon[23711]: osdmap e40: 6 total, 6 up, 6 in 2024-09-18T07:23:08.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:08 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:23:09.407 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:23:09.428 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi096:vg_nvme/lv_2 2024-09-18T07:23:09.845 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:09 smithi096 ceph-mon[25928]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:09.845 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:09 smithi096 ceph-mon[25928]: osdmap e41: 6 total, 6 up, 6 in 2024-09-18T07:23:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:09 smithi083 ceph-mon[23711]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-18T07:23:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:09 smithi083 ceph-mon[23711]: osdmap e41: 6 total, 6 up, 6 in 2024-09-18T07:23:11.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:11 smithi083 ceph-mon[23711]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:11.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:11 smithi083 ceph-mon[23711]: osdmap e42: 6 total, 6 up, 6 in 2024-09-18T07:23:11.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:11 smithi083 ceph-mon[23711]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-18T07:23:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:11 smithi096 ceph-mon[25928]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:11 smithi096 ceph-mon[25928]: osdmap e42: 6 total, 6 up, 6 in 2024-09-18T07:23:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:11 smithi096 ceph-mon[25928]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: osdmap e43: 6 total, 6 up, 6 in 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:12.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:12.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: osdmap e43: 6 total, 6 up, 6 in 2024-09-18T07:23:12.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:12.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:12.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:12.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:12.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:12.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:12.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:12.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:12.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:13 smithi083 ceph-mon[23711]: Detected new or changed devices on smithi096 2024-09-18T07:23:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:13 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi096 to 7968M 2024-09-18T07:23:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:13 smithi083 ceph-mon[23711]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:13 smithi083 ceph-mon[23711]: osdmap e44: 6 total, 6 up, 6 in 2024-09-18T07:23:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:13 smithi083 ceph-mon[23711]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:23:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:13 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:23:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:13 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:23:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:13 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:13.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:13 smithi096 ceph-mon[25928]: Detected new or changed devices on smithi096 2024-09-18T07:23:13.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:13 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi096 to 7968M 2024-09-18T07:23:13.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:13 smithi096 ceph-mon[25928]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:13.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:13 smithi096 ceph-mon[25928]: osdmap e44: 6 total, 6 up, 6 in 2024-09-18T07:23:13.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:13 smithi096 ceph-mon[25928]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:23:13.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:13 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:23:13.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:13 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:23:13.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:13 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: Cluster is now healthy 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: from='client.? 172.21.15.96:0/3396903969' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "20b8a112-7895-4837-aff1-e56761d684e2"}]: dispatch 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "20b8a112-7895-4837-aff1-e56761d684e2"}]: dispatch 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "20b8a112-7895-4837-aff1-e56761d684e2"}]': finished 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: osdmap e45: 7 total, 6 up, 7 in 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:15 smithi096 ceph-mon[25928]: from='client.? 172.21.15.96:0/2591440865' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: Cluster is now healthy 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/3396903969' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "20b8a112-7895-4837-aff1-e56761d684e2"}]: dispatch 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "20b8a112-7895-4837-aff1-e56761d684e2"}]: dispatch 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "20b8a112-7895-4837-aff1-e56761d684e2"}]': finished 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: osdmap e45: 7 total, 6 up, 7 in 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:15 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/2591440865' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:23:17.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:17 smithi083 ceph-mon[23711]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-18T07:23:17.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:17 smithi096 ceph-mon[25928]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-18T07:23:19.697 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:19 smithi096 ceph-mon[25928]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-18T07:23:19.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:19 smithi083 ceph-mon[23711]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-18T07:23:21.755 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:21 smithi096 ceph-mon[25928]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-18T07:23:21.756 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:21 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-18T07:23:21.756 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:21 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:21.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:21 smithi083 ceph-mon[23711]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-18T07:23:21.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:21 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-18T07:23:21.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:21 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:22.751 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:22 smithi096 ceph-mon[25928]: Deploying daemon osd.6 on smithi096 2024-09-18T07:23:22.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:22 smithi083 ceph-mon[23711]: Deploying daemon osd.6 on smithi096 2024-09-18T07:23:23.841 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:23 smithi096 ceph-mon[25928]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-18T07:23:23.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:23 smithi083 ceph-mon[23711]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-18T07:23:25.731 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:25 smithi096 ceph-mon[25928]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:25.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:25 smithi083 ceph-mon[23711]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:27.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:27.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:27.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:27.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:27 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:27.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:27 smithi083 ceph-mon[23711]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:27.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:27.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:27.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:27.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:27 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:27.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:27 smithi096 ceph-mon[25928]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:29.616 INFO:teuthology.orchestra.run.smithi096.stdout:Created osd(s) 6 on host 'smithi096' 2024-09-18T07:23:29.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:29 smithi083 ceph-mon[23711]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:29.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:29 smithi083 ceph-mon[23711]: from='osd.6 [v2:172.21.15.96:6816/344891191,v1:172.21.15.96:6817/344891191]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-18T07:23:29.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:29 smithi083 ceph-mon[23711]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-18T07:23:29.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:29.869 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:29 smithi096 ceph-mon[25928]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:29.870 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:29 smithi096 ceph-mon[25928]: from='osd.6 [v2:172.21.15.96:6816/344891191,v1:172.21.15.96:6817/344891191]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-18T07:23:29.870 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:29 smithi096 ceph-mon[25928]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-18T07:23:29.870 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:30.167 DEBUG:teuthology.orchestra.run.smithi096:osd.6> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.6.service 2024-09-18T07:23:30.169 INFO:tasks.cephadm:Deploying osd.7 on smithi096 with /dev/vg_nvme/lv_1... 2024-09-18T07:23:30.169 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-18T07:23:30.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:30 smithi083 ceph-mon[23711]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-18T07:23:30.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:30 smithi083 ceph-mon[23711]: osdmap e46: 7 total, 6 up, 7 in 2024-09-18T07:23:30.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:30 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:30.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:30 smithi083 ceph-mon[23711]: from='osd.6 [v2:172.21.15.96:6816/344891191,v1:172.21.15.96:6817/344891191]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:30.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:30 smithi083 ceph-mon[23711]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:30.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:30 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:30 smithi096 ceph-mon[25928]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-18T07:23:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:30 smithi096 ceph-mon[25928]: osdmap e46: 7 total, 6 up, 7 in 2024-09-18T07:23:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:30 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:30 smithi096 ceph-mon[25928]: from='osd.6 [v2:172.21.15.96:6816/344891191,v1:172.21.15.96:6817/344891191]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:30 smithi096 ceph-mon[25928]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:30 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:30.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:23:30 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[36041]: 2024-09-18T07:23:30.599+0000 7f5ebc78f700 -1 osd.6 0 waiting for initial osdmap 2024-09-18T07:23:30.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:23:30 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[36041]: 2024-09-18T07:23:30.606+0000 7f5eb5122700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:23:31.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:31 smithi096 ceph-mon[25928]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:31.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:31 smithi096 ceph-mon[25928]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-18T07:23:31.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:31 smithi096 ceph-mon[25928]: osdmap e47: 7 total, 6 up, 7 in 2024-09-18T07:23:31.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:31 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:31.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:31 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:31.729 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:31 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:31.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:31 smithi083 ceph-mon[23711]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:31.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:31 smithi083 ceph-mon[23711]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-18T07:23:31.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:31 smithi083 ceph-mon[23711]: osdmap e47: 7 total, 6 up, 7 in 2024-09-18T07:23:31.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:31 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:31.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:31 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:31.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:31 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:32.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:32 smithi083 ceph-mon[23711]: purged_snaps scrub starts 2024-09-18T07:23:32.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:32 smithi083 ceph-mon[23711]: purged_snaps scrub ok 2024-09-18T07:23:32.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:32 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:32.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:32 smithi083 ceph-mon[23711]: osd.6 [v2:172.21.15.96:6816/344891191,v1:172.21.15.96:6817/344891191] boot 2024-09-18T07:23:32.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:32 smithi083 ceph-mon[23711]: osdmap e48: 7 total, 7 up, 7 in 2024-09-18T07:23:32.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:32 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:32 smithi096 ceph-mon[25928]: purged_snaps scrub starts 2024-09-18T07:23:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:32 smithi096 ceph-mon[25928]: purged_snaps scrub ok 2024-09-18T07:23:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:32 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:32.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:32 smithi096 ceph-mon[25928]: osd.6 [v2:172.21.15.96:6816/344891191,v1:172.21.15.96:6817/344891191] boot 2024-09-18T07:23:32.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:32 smithi096 ceph-mon[25928]: osdmap e48: 7 total, 7 up, 7 in 2024-09-18T07:23:32.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:32 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:23:33.784 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:33 smithi096 ceph-mon[25928]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:33.784 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:33 smithi096 ceph-mon[25928]: osdmap e49: 7 total, 7 up, 7 in 2024-09-18T07:23:33.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:33 smithi083 ceph-mon[23711]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-18T07:23:33.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:33 smithi083 ceph-mon[23711]: osdmap e49: 7 total, 7 up, 7 in 2024-09-18T07:23:34.089 INFO:teuthology.orchestra.run.smithi096.stdout: 2024-09-18T07:23:34.114 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi096:vg_nvme/lv_1 2024-09-18T07:23:34.948 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:34 smithi096 ceph-mon[25928]: osdmap e50: 7 total, 7 up, 7 in 2024-09-18T07:23:35.117 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:34 smithi083 ceph-mon[23711]: osdmap e50: 7 total, 7 up, 7 in 2024-09-18T07:23:35.997 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:35 smithi096 ceph-mon[25928]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-18T07:23:35.997 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:35 smithi096 ceph-mon[25928]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-18T07:23:35.997 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:35 smithi096 ceph-mon[25928]: osdmap e51: 7 total, 7 up, 7 in 2024-09-18T07:23:36.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:35 smithi083 ceph-mon[23711]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-18T07:23:36.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:35 smithi083 ceph-mon[23711]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-18T07:23:36.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:35 smithi083 ceph-mon[23711]: osdmap e51: 7 total, 7 up, 7 in 2024-09-18T07:23:36.957 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: osdmap e52: 7 total, 7 up, 7 in 2024-09-18T07:23:36.957 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:36.957 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:36.958 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:36.958 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:36.958 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:36.958 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:36.958 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:36.958 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:36.958 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:36.958 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:36 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: osdmap e52: 7 total, 7 up, 7 in 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:37.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:36 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:37 smithi096 ceph-mon[25928]: Detected new or changed devices on smithi096 2024-09-18T07:23:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:37 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi096 to 5312M 2024-09-18T07:23:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:37 smithi096 ceph-mon[25928]: pgmap v156: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-18T07:23:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:37 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:23:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:37 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:23:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:37 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:38.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:37 smithi083 ceph-mon[23711]: Detected new or changed devices on smithi096 2024-09-18T07:23:38.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:37 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi096 to 5312M 2024-09-18T07:23:38.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:37 smithi083 ceph-mon[23711]: pgmap v156: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-18T07:23:38.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:37 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-18T07:23:38.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:37 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-18T07:23:38.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:37 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:39.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:38 smithi083 ceph-mon[23711]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:23:39.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:38 smithi096 ceph-mon[25928]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi096:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:23:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:39 smithi083 ceph-mon[23711]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-18T07:23:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:39 smithi083 ceph-mon[23711]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-18T07:23:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:39 smithi083 ceph-mon[23711]: Cluster is now healthy 2024-09-18T07:23:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:39 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/2066763341' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1b3bed55-4b6e-4eb1-89b5-d3d7dd7e00e8"}]: dispatch 2024-09-18T07:23:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:39 smithi083 ceph-mon[23711]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1b3bed55-4b6e-4eb1-89b5-d3d7dd7e00e8"}]: dispatch 2024-09-18T07:23:39.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:39 smithi083 ceph-mon[23711]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1b3bed55-4b6e-4eb1-89b5-d3d7dd7e00e8"}]': finished 2024-09-18T07:23:39.915 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:39 smithi083 ceph-mon[23711]: osdmap e53: 8 total, 7 up, 8 in 2024-09-18T07:23:39.915 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:39 smithi096 ceph-mon[25928]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-18T07:23:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:39 smithi096 ceph-mon[25928]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-18T07:23:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:39 smithi096 ceph-mon[25928]: Cluster is now healthy 2024-09-18T07:23:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:39 smithi096 ceph-mon[25928]: from='client.? 172.21.15.96:0/2066763341' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1b3bed55-4b6e-4eb1-89b5-d3d7dd7e00e8"}]: dispatch 2024-09-18T07:23:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:39 smithi096 ceph-mon[25928]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1b3bed55-4b6e-4eb1-89b5-d3d7dd7e00e8"}]: dispatch 2024-09-18T07:23:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:39 smithi096 ceph-mon[25928]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1b3bed55-4b6e-4eb1-89b5-d3d7dd7e00e8"}]': finished 2024-09-18T07:23:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:39 smithi096 ceph-mon[25928]: osdmap e53: 8 total, 7 up, 8 in 2024-09-18T07:23:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:41.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:40 smithi083 ceph-mon[23711]: from='client.? 172.21.15.96:0/3414300814' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:23:41.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:40 smithi096 ceph-mon[25928]: from='client.? 172.21.15.96:0/3414300814' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-18T07:23:42.092 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:41 smithi096 ceph-mon[25928]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-18T07:23:42.117 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:41 smithi083 ceph-mon[23711]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-18T07:23:43.811 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:43 smithi096 ceph-mon[25928]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-18T07:23:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:43 smithi083 ceph-mon[23711]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-18T07:23:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:23:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:23:45.117 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:23:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:23:46.037 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:45 smithi096 ceph-mon[25928]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-18T07:23:46.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:45 smithi083 ceph-mon[23711]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-18T07:23:46.949 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-18T07:23:46.949 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:46.949 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:46 smithi096 ceph-mon[25928]: Deploying daemon osd.7 on smithi096 2024-09-18T07:23:47.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-18T07:23:47.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:47.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:46 smithi083 ceph-mon[23711]: Deploying daemon osd.7 on smithi096 2024-09-18T07:23:47.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:47 smithi096 ceph-mon[25928]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-18T07:23:48.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:47 smithi083 ceph-mon[23711]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-18T07:23:49.837 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:49 smithi096 ceph-mon[25928]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:49.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:49 smithi083 ceph-mon[23711]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:51.722 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:51 smithi096 ceph-mon[25928]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:51.722 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:51 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:51.722 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:51 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:51.722 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:51 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:51.722 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:51 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:51.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:51 smithi083 ceph-mon[23711]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:51.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:51.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:51.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:51.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:51 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:53.867 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:53 smithi096 ceph-mon[25928]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:53.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:53 smithi083 ceph-mon[23711]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:54.267 INFO:teuthology.orchestra.run.smithi096.stdout:Created osd(s) 7 on host 'smithi096' 2024-09-18T07:23:54.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:54 smithi083 ceph-mon[23711]: from='osd.7 [v2:172.21.15.96:6824/14302277,v1:172.21.15.96:6825/14302277]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-18T07:23:54.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:54 smithi083 ceph-mon[23711]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-18T07:23:54.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:54 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:54.870 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:54 smithi096 ceph-mon[25928]: from='osd.7 [v2:172.21.15.96:6824/14302277,v1:172.21.15.96:6825/14302277]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-18T07:23:54.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:54 smithi096 ceph-mon[25928]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-18T07:23:54.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:54 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:54.894 DEBUG:teuthology.orchestra.run.smithi096:osd.7> sudo journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.7.service 2024-09-18T07:23:54.896 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-18T07:23:54.896 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-18T07:23:55.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:55 smithi083 ceph-mon[23711]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:55.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:55 smithi083 ceph-mon[23711]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-18T07:23:55.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:55 smithi083 ceph-mon[23711]: osdmap e54: 8 total, 7 up, 8 in 2024-09-18T07:23:55.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:55 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:55.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:55 smithi083 ceph-mon[23711]: from='osd.7 [v2:172.21.15.96:6824/14302277,v1:172.21.15.96:6825/14302277]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:55.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:55 smithi083 ceph-mon[23711]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:55.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:55 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:55.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:55 smithi096 ceph-mon[25928]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:55.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:55 smithi096 ceph-mon[25928]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-18T07:23:55.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:55 smithi096 ceph-mon[25928]: osdmap e54: 8 total, 7 up, 8 in 2024-09-18T07:23:55.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:55 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:55.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:55 smithi096 ceph-mon[25928]: from='osd.7 [v2:172.21.15.96:6824/14302277,v1:172.21.15.96:6825/14302277]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:55.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:55 smithi096 ceph-mon[25928]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]: dispatch 2024-09-18T07:23:55.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:55 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:55.873 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:23:55 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[39026]: 2024-09-18T07:23:55.598+0000 7fef73786700 -1 osd.7 0 waiting for initial osdmap 2024-09-18T07:23:55.873 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:23:55 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[39026]: 2024-09-18T07:23:55.605+0000 7fef6e91e700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:23:56.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:56 smithi083 ceph-mon[23711]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-18T07:23:56.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:56 smithi083 ceph-mon[23711]: osdmap e55: 8 total, 7 up, 8 in 2024-09-18T07:23:56.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:56 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:56.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:56 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:56.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:56 smithi096 ceph-mon[25928]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]}]': finished 2024-09-18T07:23:56.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:56 smithi096 ceph-mon[25928]: osdmap e55: 8 total, 7 up, 8 in 2024-09-18T07:23:56.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:56 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:56.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:56 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:56.909 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:23:57.591 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1726644236,"num_in_osds":8,"osd_in_since":1726644219,"num_remapped_pgs":1} 2024-09-18T07:23:57.592 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-18T07:23:57.803 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:23:57.834 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:57 smithi096 ceph-mon[25928]: purged_snaps scrub starts 2024-09-18T07:23:57.834 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:57 smithi096 ceph-mon[25928]: purged_snaps scrub ok 2024-09-18T07:23:57.834 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:57 smithi096 ceph-mon[25928]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:57.834 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:57 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:57.835 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:57 smithi096 ceph-mon[25928]: osd.7 [v2:172.21.15.96:6824/14302277,v1:172.21.15.96:6825/14302277] boot 2024-09-18T07:23:57.835 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:57 smithi096 ceph-mon[25928]: osdmap e56: 8 total, 8 up, 8 in 2024-09-18T07:23:57.835 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:57 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:57.835 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:57 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1131329995' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-18T07:23:57.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:57 smithi083 ceph-mon[23711]: purged_snaps scrub starts 2024-09-18T07:23:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:57 smithi083 ceph-mon[23711]: purged_snaps scrub ok 2024-09-18T07:23:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:57 smithi083 ceph-mon[23711]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-18T07:23:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:57 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:57 smithi083 ceph-mon[23711]: osd.7 [v2:172.21.15.96:6824/14302277,v1:172.21.15.96:6825/14302277] boot 2024-09-18T07:23:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:57 smithi083 ceph-mon[23711]: osdmap e56: 8 total, 8 up, 8 in 2024-09-18T07:23:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:57 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:23:57.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:57 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1131329995' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-18T07:23:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: osdmap e57: 8 total, 8 up, 8 in 2024-09-18T07:23:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:58.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:58.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:58.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:58.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:58.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:58.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:58.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: osdmap e57: 8 total, 8 up, 8 in 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:23:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:23:59.632 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:23:59.632 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":59,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","created":"2024-09-18T07:17:55.270255+0000","modified":"2024-09-18T07:23:59.606765+0000","last_up_change":"2024-09-18T07:23:56.596204+0000","last_in_change":"2024-09-18T07:23:39.262896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T07:21:58.791682+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fb38d4aa-4e71-4a92-9d7e-a3f5df569ab1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6803","nonce":3200079573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6805","nonce":3200079573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6809","nonce":3200079573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6807","nonce":3200079573}]},"public_addr":"172.21.15.83:6803/3200079573","cluster_addr":"172.21.15.83:6805/3200079573","heartbeat_back_addr":"172.21.15.83:6809/3200079573","heartbeat_front_addr":"172.21.15.83:6807/3200079573","state":["exists","up"]},{"osd":1,"uuid":"e6e470d9-3890-49fa-81a4-e017a796053e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6811","nonce":24696401}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6813","nonce":24696401}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6817","nonce":24696401}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6815","nonce":24696401}]},"public_addr":"172.21.15.83:6811/24696401","cluster_addr":"172.21.15.83:6813/24696401","heartbeat_back_addr":"172.21.15.83:6817/24696401","heartbeat_front_addr":"172.21.15.83:6815/24696401","state":["exists","up"]},{"osd":2,"uuid":"b104b18e-f2d0-4a92-86d6-068c62835a6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6819","nonce":1158890812}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6821","nonce":1158890812}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6825","nonce":1158890812}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6823","nonce":1158890812}]},"public_addr":"172.21.15.83:6819/1158890812","cluster_addr":"172.21.15.83:6821/1158890812","heartbeat_back_addr":"172.21.15.83:6825/1158890812","heartbeat_front_addr":"172.21.15.83:6823/1158890812","state":["exists","up"]},{"osd":3,"uuid":"48c6e315-85de-413e-b9a2-8a067af7a090","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6827","nonce":4086177851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6829","nonce":4086177851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6833","nonce":4086177851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6831","nonce":4086177851}]},"public_addr":"172.21.15.83:6827/4086177851","cluster_addr":"172.21.15.83:6829/4086177851","heartbeat_back_addr":"172.21.15.83:6833/4086177851","heartbeat_front_addr":"172.21.15.83:6831/4086177851","state":["exists","up"]},{"osd":4,"uuid":"e0cef4e7-db42-4b46-a43e-dc596efcc266","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6800","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6801","nonce":2765851169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6803","nonce":2765851169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6807","nonce":2765851169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6805","nonce":2765851169}]},"public_addr":"172.21.15.96:6801/2765851169","cluster_addr":"172.21.15.96:6803/2765851169","heartbeat_back_addr":"172.21.15.96:6807/2765851169","heartbeat_front_addr":"172.21.15.96:6805/2765851169","state":["exists","up"]},{"osd":5,"uuid":"dcc03009-1d9f-4b1b-84ec-0be59727452c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6809","nonce":692248156}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6811","nonce":692248156}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6815","nonce":692248156}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6813","nonce":692248156}]},"public_addr":"172.21.15.96:6809/692248156","cluster_addr":"172.21.15.96:6811/692248156","heartbeat_back_addr":"172.21.15.96:6815/692248156","heartbeat_front_addr":"172.21.15.96:6813/692248156","state":["exists","up"]},{"osd":6,"uuid":"20b8a112-7895-4837-aff1-e56761d684e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6817","nonce":344891191}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6819","nonce":344891191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6823","nonce":344891191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6821","nonce":344891191}]},"public_addr":"172.21.15.96:6817/344891191","cluster_addr":"172.21.15.96:6819/344891191","heartbeat_back_addr":"172.21.15.96:6823/344891191","heartbeat_front_addr":"172.21.15.96:6821/344891191","state":["exists","up"]},{"osd":7,"uuid":"1b3bed55-4b6e-4eb1-89b5-d3d7dd7e00e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6825","nonce":14302277}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6827","nonce":14302277}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6831","nonce":14302277}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6829","nonce":14302277}]},"public_addr":"172.21.15.96:6825/14302277","cluster_addr":"172.21.15.96:6827/14302277","heartbeat_back_addr":"172.21.15.96:6831/14302277","heartbeat_front_addr":"172.21.15.96:6829/14302277","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:20:56.560539+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:21:28.577023+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:21:55.410857+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:22:19.815754+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:22:41.470140+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:23:06.620517+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:23:30.418037+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:23:54.886847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6800/2957183686":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/3416035082":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/2698291578":"2024-09-19T07:19:43.644203+0000","172.21.15.83:6800/1953836661":"2024-09-19T07:18:56.327964+0000","172.21.15.83:0/2518140709":"2024-09-19T07:18:56.327964+0000","172.21.15.83:0/496590967":"2024-09-19T07:18:56.327964+0000","172.21.15.83:6800/2711710262":"2024-09-19T07:18:28.179241+0000","172.21.15.83:6801/1953836661":"2024-09-19T07:18:56.327964+0000","172.21.15.83:6801/2711710262":"2024-09-19T07:18:28.179241+0000","172.21.15.83:6801/2957183686":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/944422047":"2024-09-19T07:18:28.179241+0000","172.21.15.83:0/2325968748":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/1100990555":"2024-09-19T07:18:56.327964+0000","172.21.15.83:0/3134518866":"2024-09-19T07:18:28.179241+0000","172.21.15.83:0/2395847317":"2024-09-19T07:18:28.179241+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T07:23:59.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:59 smithi083 ceph-mon[23711]: Detected new or changed devices on smithi096 2024-09-18T07:23:59.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:59 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi096 to 3984M 2024-09-18T07:23:59.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:59 smithi083 ceph-mon[23711]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:23:59.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:23:59 smithi083 ceph-mon[23711]: osdmap e58: 8 total, 8 up, 8 in 2024-09-18T07:23:59.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:59 smithi096 ceph-mon[25928]: Detected new or changed devices on smithi096 2024-09-18T07:23:59.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:59 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi096 to 3984M 2024-09-18T07:23:59.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:59 smithi096 ceph-mon[25928]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:23:59.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:23:59 smithi096 ceph-mon[25928]: osdmap e58: 8 total, 8 up, 8 in 2024-09-18T07:24:00.294 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-18T07:21:58.791682+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-18T07:24:00.295 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-18T07:24:00.505 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:00 smithi083 ceph-mon[23711]: osdmap e59: 8 total, 8 up, 8 in 2024-09-18T07:24:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:00 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2827695247' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-18T07:24:00.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:00 smithi096 ceph-mon[25928]: osdmap e59: 8 total, 8 up, 8 in 2024-09-18T07:24:00.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:00 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2827695247' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-18T07:24:01.824 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:01 smithi083 ceph-mon[23711]: pgmap v175: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-18T07:24:01.824 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:01 smithi083 ceph-mon[23711]: osdmap e60: 8 total, 8 up, 8 in 2024-09-18T07:24:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:01 smithi096 ceph-mon[25928]: pgmap v175: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-18T07:24:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:01 smithi096 ceph-mon[25928]: osdmap e60: 8 total, 8 up, 8 in 2024-09-18T07:24:02.234 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 1 2024-09-18T07:24:02.942 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:02 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/430460701' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-18T07:24:02.968 INFO:tasks.cephadm:Setting up client nodes... 2024-09-18T07:24:02.968 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-18T07:24:03.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:02 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/430460701' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-18T07:24:03.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:03 smithi083 ceph-mon[23711]: pgmap v177: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-18T07:24:04.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:03 smithi096 ceph-mon[25928]: pgmap v177: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-18T07:24:05.005 INFO:teuthology.orchestra.run.smithi083.stdout:[client.0] 2024-09-18T07:24:05.005 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQAUgOpmIXmDOxAAvT3d2ptNo2Vnaxq2n4BGjA== 2024-09-18T07:24:05.613 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-09-18T07:24:05.613 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-18T07:24:05.613 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-18T07:24:05.652 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-18T07:24:05.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:05 smithi083 ceph-mon[23711]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-18T07:24:05.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:05 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2057563380' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-18T07:24:05.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:05 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2057563380' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-18T07:24:05.895 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:06.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:05 smithi096 ceph-mon[25928]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-18T07:24:06.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:05 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2057563380' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-18T07:24:06.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:05 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2057563380' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-18T07:24:07.714 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:24:08.011 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:07 smithi083 ceph-mon[23711]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-18T07:24:08.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:07 smithi096 ceph-mon[25928]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-18T07:24:08.354 INFO:teuthology.orchestra.run.smithi083.stdout:[{"version":21,"timestamp":"2024-09-18T07:23:57.865665+0000","name":"","changes":[{"name":"osd/host:smithi096/osd_memory_target","previous_value":"5570072166","new_value":"4177554124"}]}] 2024-09-18T07:24:08.354 INFO:tasks.ceph_manager:config epoch is 21 2024-09-18T07:24:08.354 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-18T07:24:08.354 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-18T07:24:08.355 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-18T07:24:08.561 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:08.982 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:08 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/4290854968' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-18T07:24:09.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:08 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/4290854968' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-18T07:24:09.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:09 smithi083 ceph-mon[23711]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:10.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:09 smithi096 ceph-mon[25928]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:10.357 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:24:10.934 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:10 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/873740980' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-18T07:24:10.957 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi083.vchvnu","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6800","nonce":1065123569},{"type":"v1","addr":"172.21.15.83:6801","nonce":1065123569}]},"active_addr":"172.21.15.83:6801/1065123569","active_change":"2024-09-18T07:19:43.644532+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi096.caifnv","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.83:8443/","prometheus":"http://172.21.15.83:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":1318154783}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3424919341}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":2167531824}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":4008072026}]}]}} 2024-09-18T07:24:10.961 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-18T07:24:10.961 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-18T07:24:10.961 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-18T07:24:11.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:10 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/873740980' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-18T07:24:11.167 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:11.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:11 smithi083 ceph-mon[23711]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:12.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:11 smithi096 ceph-mon[25928]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:13.012 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:24:13.013 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":60,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","created":"2024-09-18T07:17:55.270255+0000","modified":"2024-09-18T07:24:00.613027+0000","last_up_change":"2024-09-18T07:23:56.596204+0000","last_in_change":"2024-09-18T07:23:39.262896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T07:21:58.791682+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fb38d4aa-4e71-4a92-9d7e-a3f5df569ab1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6803","nonce":3200079573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6805","nonce":3200079573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6809","nonce":3200079573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6807","nonce":3200079573}]},"public_addr":"172.21.15.83:6803/3200079573","cluster_addr":"172.21.15.83:6805/3200079573","heartbeat_back_addr":"172.21.15.83:6809/3200079573","heartbeat_front_addr":"172.21.15.83:6807/3200079573","state":["exists","up"]},{"osd":1,"uuid":"e6e470d9-3890-49fa-81a4-e017a796053e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6811","nonce":24696401}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6813","nonce":24696401}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6817","nonce":24696401}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6815","nonce":24696401}]},"public_addr":"172.21.15.83:6811/24696401","cluster_addr":"172.21.15.83:6813/24696401","heartbeat_back_addr":"172.21.15.83:6817/24696401","heartbeat_front_addr":"172.21.15.83:6815/24696401","state":["exists","up"]},{"osd":2,"uuid":"b104b18e-f2d0-4a92-86d6-068c62835a6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6819","nonce":1158890812}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6821","nonce":1158890812}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6825","nonce":1158890812}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6823","nonce":1158890812}]},"public_addr":"172.21.15.83:6819/1158890812","cluster_addr":"172.21.15.83:6821/1158890812","heartbeat_back_addr":"172.21.15.83:6825/1158890812","heartbeat_front_addr":"172.21.15.83:6823/1158890812","state":["exists","up"]},{"osd":3,"uuid":"48c6e315-85de-413e-b9a2-8a067af7a090","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6827","nonce":4086177851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6829","nonce":4086177851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6833","nonce":4086177851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6831","nonce":4086177851}]},"public_addr":"172.21.15.83:6827/4086177851","cluster_addr":"172.21.15.83:6829/4086177851","heartbeat_back_addr":"172.21.15.83:6833/4086177851","heartbeat_front_addr":"172.21.15.83:6831/4086177851","state":["exists","up"]},{"osd":4,"uuid":"e0cef4e7-db42-4b46-a43e-dc596efcc266","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6800","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6801","nonce":2765851169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6803","nonce":2765851169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6807","nonce":2765851169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6805","nonce":2765851169}]},"public_addr":"172.21.15.96:6801/2765851169","cluster_addr":"172.21.15.96:6803/2765851169","heartbeat_back_addr":"172.21.15.96:6807/2765851169","heartbeat_front_addr":"172.21.15.96:6805/2765851169","state":["exists","up"]},{"osd":5,"uuid":"dcc03009-1d9f-4b1b-84ec-0be59727452c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6809","nonce":692248156}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6811","nonce":692248156}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6815","nonce":692248156}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6813","nonce":692248156}]},"public_addr":"172.21.15.96:6809/692248156","cluster_addr":"172.21.15.96:6811/692248156","heartbeat_back_addr":"172.21.15.96:6815/692248156","heartbeat_front_addr":"172.21.15.96:6813/692248156","state":["exists","up"]},{"osd":6,"uuid":"20b8a112-7895-4837-aff1-e56761d684e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6817","nonce":344891191}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6819","nonce":344891191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6823","nonce":344891191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6821","nonce":344891191}]},"public_addr":"172.21.15.96:6817/344891191","cluster_addr":"172.21.15.96:6819/344891191","heartbeat_back_addr":"172.21.15.96:6823/344891191","heartbeat_front_addr":"172.21.15.96:6821/344891191","state":["exists","up"]},{"osd":7,"uuid":"1b3bed55-4b6e-4eb1-89b5-d3d7dd7e00e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6825","nonce":14302277}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6827","nonce":14302277}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6831","nonce":14302277}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6829","nonce":14302277}]},"public_addr":"172.21.15.96:6825/14302277","cluster_addr":"172.21.15.96:6827/14302277","heartbeat_back_addr":"172.21.15.96:6831/14302277","heartbeat_front_addr":"172.21.15.96:6829/14302277","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:20:56.560539+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:21:28.577023+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:21:55.410857+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:22:19.815754+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:22:41.470140+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:23:06.620517+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:23:30.418037+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:23:54.886847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6800/2957183686":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/3416035082":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/2698291578":"2024-09-19T07:19:43.644203+0000","172.21.15.83:6800/1953836661":"2024-09-19T07:18:56.327964+0000","172.21.15.83:0/2518140709":"2024-09-19T07:18:56.327964+0000","172.21.15.83:0/496590967":"2024-09-19T07:18:56.327964+0000","172.21.15.83:6800/2711710262":"2024-09-19T07:18:28.179241+0000","172.21.15.83:6801/1953836661":"2024-09-19T07:18:56.327964+0000","172.21.15.83:6801/2711710262":"2024-09-19T07:18:28.179241+0000","172.21.15.83:6801/2957183686":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/944422047":"2024-09-19T07:18:28.179241+0000","172.21.15.83:0/2325968748":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/1100990555":"2024-09-19T07:18:56.327964+0000","172.21.15.83:0/3134518866":"2024-09-19T07:18:28.179241+0000","172.21.15.83:0/2395847317":"2024-09-19T07:18:28.179241+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T07:24:13.698 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-18T07:24:13.698 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-18T07:24:13.904 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:14.024 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:13 smithi083 ceph-mon[23711]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:14.025 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:13 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/4202128839' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-18T07:24:14.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:13 smithi096 ceph-mon[25928]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:14.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:13 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/4202128839' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-18T07:24:15.600 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:24:15.600 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":60,"fsid":"126f471a-758e-11ef-bceb-c7b262605968","created":"2024-09-18T07:17:55.270255+0000","modified":"2024-09-18T07:24:00.613027+0000","last_up_change":"2024-09-18T07:23:56.596204+0000","last_in_change":"2024-09-18T07:23:39.262896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T07:21:58.791682+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fb38d4aa-4e71-4a92-9d7e-a3f5df569ab1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6803","nonce":3200079573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6805","nonce":3200079573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6809","nonce":3200079573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":3200079573},{"type":"v1","addr":"172.21.15.83:6807","nonce":3200079573}]},"public_addr":"172.21.15.83:6803/3200079573","cluster_addr":"172.21.15.83:6805/3200079573","heartbeat_back_addr":"172.21.15.83:6809/3200079573","heartbeat_front_addr":"172.21.15.83:6807/3200079573","state":["exists","up"]},{"osd":1,"uuid":"e6e470d9-3890-49fa-81a4-e017a796053e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6811","nonce":24696401}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6813","nonce":24696401}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6817","nonce":24696401}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":24696401},{"type":"v1","addr":"172.21.15.83:6815","nonce":24696401}]},"public_addr":"172.21.15.83:6811/24696401","cluster_addr":"172.21.15.83:6813/24696401","heartbeat_back_addr":"172.21.15.83:6817/24696401","heartbeat_front_addr":"172.21.15.83:6815/24696401","state":["exists","up"]},{"osd":2,"uuid":"b104b18e-f2d0-4a92-86d6-068c62835a6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6819","nonce":1158890812}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6821","nonce":1158890812}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6825","nonce":1158890812}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":1158890812},{"type":"v1","addr":"172.21.15.83:6823","nonce":1158890812}]},"public_addr":"172.21.15.83:6819/1158890812","cluster_addr":"172.21.15.83:6821/1158890812","heartbeat_back_addr":"172.21.15.83:6825/1158890812","heartbeat_front_addr":"172.21.15.83:6823/1158890812","state":["exists","up"]},{"osd":3,"uuid":"48c6e315-85de-413e-b9a2-8a067af7a090","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6827","nonce":4086177851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6829","nonce":4086177851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6833","nonce":4086177851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":4086177851},{"type":"v1","addr":"172.21.15.83:6831","nonce":4086177851}]},"public_addr":"172.21.15.83:6827/4086177851","cluster_addr":"172.21.15.83:6829/4086177851","heartbeat_back_addr":"172.21.15.83:6833/4086177851","heartbeat_front_addr":"172.21.15.83:6831/4086177851","state":["exists","up"]},{"osd":4,"uuid":"e0cef4e7-db42-4b46-a43e-dc596efcc266","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6800","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6801","nonce":2765851169}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6803","nonce":2765851169}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6807","nonce":2765851169}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":2765851169},{"type":"v1","addr":"172.21.15.96:6805","nonce":2765851169}]},"public_addr":"172.21.15.96:6801/2765851169","cluster_addr":"172.21.15.96:6803/2765851169","heartbeat_back_addr":"172.21.15.96:6807/2765851169","heartbeat_front_addr":"172.21.15.96:6805/2765851169","state":["exists","up"]},{"osd":5,"uuid":"dcc03009-1d9f-4b1b-84ec-0be59727452c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6809","nonce":692248156}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6811","nonce":692248156}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6815","nonce":692248156}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":692248156},{"type":"v1","addr":"172.21.15.96:6813","nonce":692248156}]},"public_addr":"172.21.15.96:6809/692248156","cluster_addr":"172.21.15.96:6811/692248156","heartbeat_back_addr":"172.21.15.96:6815/692248156","heartbeat_front_addr":"172.21.15.96:6813/692248156","state":["exists","up"]},{"osd":6,"uuid":"20b8a112-7895-4837-aff1-e56761d684e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6817","nonce":344891191}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6819","nonce":344891191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6823","nonce":344891191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":344891191},{"type":"v1","addr":"172.21.15.96:6821","nonce":344891191}]},"public_addr":"172.21.15.96:6817/344891191","cluster_addr":"172.21.15.96:6819/344891191","heartbeat_back_addr":"172.21.15.96:6823/344891191","heartbeat_front_addr":"172.21.15.96:6821/344891191","state":["exists","up"]},{"osd":7,"uuid":"1b3bed55-4b6e-4eb1-89b5-d3d7dd7e00e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6825","nonce":14302277}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6826","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6827","nonce":14302277}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6830","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6831","nonce":14302277}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6828","nonce":14302277},{"type":"v1","addr":"172.21.15.96:6829","nonce":14302277}]},"public_addr":"172.21.15.96:6825/14302277","cluster_addr":"172.21.15.96:6827/14302277","heartbeat_back_addr":"172.21.15.96:6831/14302277","heartbeat_front_addr":"172.21.15.96:6829/14302277","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:20:56.560539+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:21:28.577023+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:21:55.410857+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:22:19.815754+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:22:41.470140+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:23:06.620517+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:23:30.418037+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T07:23:54.886847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:6800/2957183686":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/3416035082":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/2698291578":"2024-09-19T07:19:43.644203+0000","172.21.15.83:6800/1953836661":"2024-09-19T07:18:56.327964+0000","172.21.15.83:0/2518140709":"2024-09-19T07:18:56.327964+0000","172.21.15.83:0/496590967":"2024-09-19T07:18:56.327964+0000","172.21.15.83:6800/2711710262":"2024-09-19T07:18:28.179241+0000","172.21.15.83:6801/1953836661":"2024-09-19T07:18:56.327964+0000","172.21.15.83:6801/2711710262":"2024-09-19T07:18:28.179241+0000","172.21.15.83:6801/2957183686":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/944422047":"2024-09-19T07:18:28.179241+0000","172.21.15.83:0/2325968748":"2024-09-19T07:19:43.644203+0000","172.21.15.83:0/1100990555":"2024-09-19T07:18:56.327964+0000","172.21.15.83:0/3134518866":"2024-09-19T07:18:28.179241+0000","172.21.15.83:0/2395847317":"2024-09-19T07:18:28.179241+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T07:24:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:15 smithi083 ceph-mon[23711]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:15.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:15 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3106929007' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-18T07:24:16.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:15 smithi096 ceph-mon[25928]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:16.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:15 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/3106929007' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-18T07:24:16.294 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-18T07:24:16.295 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-18T07:24:16.295 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-18T07:24:16.296 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-18T07:24:16.296 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-18T07:24:16.297 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-18T07:24:16.297 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-18T07:24:16.298 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-18T07:24:16.727 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:16.728 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:16.728 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:16.730 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:16.731 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:16.738 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:16.738 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:16.738 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:17.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:17 smithi083 ceph-mon[23711]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:18.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:17 smithi096 ceph-mon[25928]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:19.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:19 smithi083 ceph-mon[23711]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:20.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:19 smithi096 ceph-mon[25928]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:21.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:21 smithi083 ceph-mon[23711]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:21 smithi096 ceph-mon[25928]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:23.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:23 smithi083 ceph-mon[23711]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:23.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:23 smithi096 ceph-mon[25928]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:25.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:25 smithi096 ceph-mon[25928]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:25.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:25 smithi083 ceph-mon[23711]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:27.019 INFO:teuthology.orchestra.run.smithi083.stdout:115964117020 2024-09-18T07:24:27.019 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-18T07:24:27.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:27 smithi083 ceph-mon[23711]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:27.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:27 smithi096 ceph-mon[25928]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:28.303 INFO:teuthology.orchestra.run.smithi083.stdout:137438953495 2024-09-18T07:24:28.303 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-18T07:24:29.120 INFO:teuthology.orchestra.run.smithi083.stdout:38654705709 2024-09-18T07:24:29.121 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-18T07:24:29.844 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:29 smithi083 ceph-mon[23711]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:29.866 INFO:teuthology.orchestra.run.smithi083.stdout:81604378656 2024-09-18T07:24:29.867 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-18T07:24:29.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:29 smithi096 ceph-mon[25928]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:31.173 INFO:teuthology.orchestra.run.smithi083.stdout:206158430223 2024-09-18T07:24:31.173 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-18T07:24:31.767 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:31 smithi083 ceph-mon[23711]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:31.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:31 smithi096 ceph-mon[25928]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:32.560 INFO:teuthology.orchestra.run.smithi083.stdout:240518168586 2024-09-18T07:24:32.560 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-18T07:24:33.052 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:33.405 INFO:teuthology.orchestra.run.smithi083.stdout:171798691859 2024-09-18T07:24:33.406 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-18T07:24:33.667 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:33 smithi083 ceph-mon[23711]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:33.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:33 smithi096 ceph-mon[25928]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:34.537 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:34.538 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:34.540 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:34.540 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:34.541 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:35.076 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:35.097 INFO:teuthology.orchestra.run.smithi083.stdout:60129542183 2024-09-18T07:24:35.098 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-18T07:24:35.389 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:35.515 INFO:teuthology.orchestra.run.smithi083.stdout:115964117022 2024-09-18T07:24:35.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:35 smithi096 ceph-mon[25928]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:35.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:35 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/3322337193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-18T07:24:35.860 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:35 smithi083 ceph-mon[23711]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:35.861 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:35 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3322337193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-18T07:24:36.164 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117020 got 115964117022 for osd.3 2024-09-18T07:24:36.165 DEBUG:teuthology.parallel:result is None 2024-09-18T07:24:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:37 smithi083 ceph-mon[23711]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:37.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:37 smithi096 ceph-mon[25928]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:39.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:39 smithi083 ceph-mon[23711]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:39.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:39 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/4093213679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-18T07:24:39.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:39 smithi096 ceph-mon[25928]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:39.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:39 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/4093213679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-18T07:24:41.593 INFO:teuthology.orchestra.run.smithi083.stdout:137438953497 2024-09-18T07:24:41.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:41 smithi083 ceph-mon[23711]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:41.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:41 smithi096 ceph-mon[25928]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:43.098 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953495 got 137438953497 for osd.4 2024-09-18T07:24:43.098 DEBUG:teuthology.parallel:result is None 2024-09-18T07:24:43.276 INFO:teuthology.orchestra.run.smithi083.stdout:206158430225 2024-09-18T07:24:43.848 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:43 smithi083 ceph-mon[23711]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:43.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:43 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/337458583' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-18T07:24:43.850 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430223 got 206158430225 for osd.6 2024-09-18T07:24:43.850 DEBUG:teuthology.parallel:result is None 2024-09-18T07:24:43.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:43 smithi096 ceph-mon[25928]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:43.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:43 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/337458583' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-18T07:24:44.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:24:44.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:24:44.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:24:44.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:24:45.157 INFO:teuthology.orchestra.run.smithi083.stdout:240518168589 2024-09-18T07:24:45.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:45 smithi096 ceph-mon[25928]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:45 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2845733071' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-18T07:24:45.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:45 smithi083 ceph-mon[23711]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:45.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:45 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2845733071' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-18T07:24:45.765 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168586 got 240518168589 for osd.7 2024-09-18T07:24:45.766 DEBUG:teuthology.parallel:result is None 2024-09-18T07:24:47.345 INFO:teuthology.orchestra.run.smithi083.stdout:171798691862 2024-09-18T07:24:47.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:47 smithi083 ceph-mon[23711]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:47.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:47 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/12235863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-18T07:24:47.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:47 smithi096 ceph-mon[25928]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:47.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:47 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/12235863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-18T07:24:47.988 INFO:teuthology.orchestra.run.smithi083.stdout:38654705713 2024-09-18T07:24:48.428 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691859 got 171798691862 for osd.5 2024-09-18T07:24:48.428 DEBUG:teuthology.parallel:result is None 2024-09-18T07:24:48.518 INFO:teuthology.orchestra.run.smithi083.stdout:60129542187 2024-09-18T07:24:48.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:48 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1012653656' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-18T07:24:48.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:48 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1401884859' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-18T07:24:48.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:48 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1012653656' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-18T07:24:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:48 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1401884859' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-18T07:24:49.186 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542187 for osd.1 2024-09-18T07:24:49.186 DEBUG:teuthology.parallel:result is None 2024-09-18T07:24:49.585 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705713 for osd.0 2024-09-18T07:24:49.585 DEBUG:teuthology.parallel:result is None 2024-09-18T07:24:49.845 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:49 smithi083 ceph-mon[23711]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:49.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:49 smithi096 ceph-mon[25928]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:50.007 INFO:teuthology.orchestra.run.smithi083.stdout:81604378662 2024-09-18T07:24:50.617 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378656 got 81604378662 for osd.2 2024-09-18T07:24:50.618 DEBUG:teuthology.parallel:result is None 2024-09-18T07:24:50.618 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-18T07:24:50.618 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-18T07:24:50.824 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:50.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:50 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3178174724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-18T07:24:50.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:50 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/3178174724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-18T07:24:51.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:51 smithi083 ceph-mon[23711]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:51.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:51 smithi096 ceph-mon[25928]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:52.531 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:24:52.532 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-18T07:24:53.166 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-09-18T07:24:50.543593+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49008,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912208,"statfs":{"total":767960285184,"available":767910100992,"internally_reserved":0,"allocated":6537216,"data_stored":3572274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004081"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-18T07:24:00.623503+0000","last_change":"2024-09-18T07:24:00.623503+0000","last_active":"2024-09-18T07:24:00.623503+0000","last_peered":"2024-09-18T07:24:00.623503+0000","last_clean":"2024-09-18T07:24:00.623503+0000","last_became_active":"2024-09-18T07:24:00.623019+0000","last_became_peered":"2024-09-18T07:24:00.623019+0000","last_unstale":"2024-09-18T07:24:00.623503+0000","last_undegraded":"2024-09-18T07:24:00.623503+0000","last_fullsized":"2024-09-18T07:24:00.623503+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T07:21:59.049834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T07:21:59.049834+0000","last_clean_scrub_stamp":"2024-09-18T07:21:59.049834+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T10:34:03.872925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47299999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57999999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68700000000000006}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6120,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739032,"statfs":{"total":95995035648,"available":95988768768,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.56299999999999994}]},{"osd":1,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.50700000000000001}]},{"osd":2,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.45300000000000001}]},{"osd":3,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.59899999999999998}]},{"osd":4,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.57999999999999996}]},{"osd":5,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.63100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.621}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:34 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.36399999999999999,"15min":0.34399999999999997},"min":{"1min":0.30499999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.35999999999999999,"15min":0.33800000000000002},"min":{"1min":0.23100000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.58099999999999996,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.45400000000000001}]},{"osd":2,"last update":"Wed Sep 18 07:24:00 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.39600000000000002,"15min":0.38900000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.41799999999999998,"15min":0.41399999999999998},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.41399999999999998}]},{"osd":3,"last update":"Wed Sep 18 07:24:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41199999999999998,"15min":0.40699999999999997},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.56799999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.46400000000000002,"15min":0.46100000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.48699999999999999}]},{"osd":4,"last update":"Wed Sep 18 07:24:45 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.58899999999999997,"15min":0.57599999999999996},"min":{"1min":0.47899999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.61199999999999999,"15min":0.60199999999999998},"min":{"1min":0.47899999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.78000000000000003}]},{"osd":5,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.84499999999999997}]},{"osd":6,"last update":"Wed Sep 18 07:24:34 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.752},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.83099999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85499999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.33100000000000002,"15min":0.29499999999999998},"min":{"1min":0.221,"5min":0.22,"15min":0.22},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.34300000000000003,"15min":0.307},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.46100000000000002}]},{"osd":2,"last update":"Wed Sep 18 07:24:02 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.255,"5min":0.251,"15min":0.251},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.434,"5min":0.38600000000000001,"15min":0.378},"min":{"1min":0.27600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.38200000000000001}]},{"osd":3,"last update":"Wed Sep 18 07:24:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45600000000000002,"15min":0.45500000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.42199999999999999}]},{"osd":4,"last update":"Wed Sep 18 07:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.45400000000000001}]},{"osd":5,"last update":"Wed Sep 18 07:24:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.54200000000000004}]},{"osd":6,"last update":"Wed Sep 18 07:24:32 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.58599999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.34699999999999998},"min":{"1min":0.27000000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.376,"15min":0.371},"min":{"1min":0.30299999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.372}]},{"osd":1,"last update":"Wed Sep 18 07:24:06 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38500000000000001,"15min":0.38300000000000001},"min":{"1min":0.29899999999999999,"5min":0.25,"15min":0.25},"max":{"1min":0.47999999999999998,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.39400000000000002,"15min":0.38900000000000001},"min":{"1min":0.29299999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.28899999999999998}]},{"osd":3,"last update":"Wed Sep 18 07:24:23 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.36299999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.58899999999999997,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.441,"15min":0.44},"min":{"1min":0.32900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.437}]},{"osd":4,"last update":"Wed Sep 18 07:24:44 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.56799999999999995,"15min":0.55500000000000005},"min":{"1min":0.49099999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.59499999999999997,"15min":0.58399999999999996},"min":{"1min":0.499,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.61699999999999999}]},{"osd":5,"last update":"Wed Sep 18 07:24:12 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.69599999999999995}]},{"osd":6,"last update":"Wed Sep 18 07:24:31 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.65500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60899999999999999}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:28 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40300000000000002,"15min":0.40400000000000003},"min":{"1min":0.27300000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.50600000000000001,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42899999999999999,"15min":0.42999999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.28599999999999998}]},{"osd":1,"last update":"Wed Sep 18 07:24:28 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.42099999999999999,"15min":0.41799999999999998},"min":{"1min":0.32700000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42899999999999999,"15min":0.42999999999999999},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.373}]},{"osd":2,"last update":"Wed Sep 18 07:24:28 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.38,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.438,"15min":0.441},"min":{"1min":0.315,"5min":0.312,"15min":0.312},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.318}]},{"osd":4,"last update":"Wed Sep 18 07:24:46 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.56799999999999995,"15min":0.55800000000000005},"min":{"1min":0.46500000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.56599999999999995,"15min":0.55900000000000005},"min":{"1min":0.44400000000000001,"5min":0.441,"15min":0.441},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.59099999999999997}]},{"osd":5,"last update":"Wed Sep 18 07:24:09 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.76300000000000001}]},{"osd":6,"last update":"Wed Sep 18 07:24:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.67700000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60299999999999998}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5796,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739356,"statfs":{"total":95995035648,"available":95989100544,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.504,"15min":0.5},"min":{"1min":0.36899999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51700000000000002,"15min":0.51500000000000001},"min":{"1min":0.36599999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.59999999999999998}]},{"osd":1,"last update":"Wed Sep 18 07:24:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54100000000000004,"15min":0.54400000000000004},"min":{"1min":0.39200000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.52800000000000002,"15min":0.52200000000000002},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.71299999999999997}]},{"osd":2,"last update":"Wed Sep 18 07:24:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.53400000000000003,"15min":0.53500000000000003},"min":{"1min":0.41099999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.76700000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52100000000000002,"15min":0.51900000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.70999999999999996,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.64900000000000002}]},{"osd":3,"last update":"Wed Sep 18 07:24:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51600000000000001,"15min":0.51400000000000001},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.623},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.51700000000000002,"15min":0.51500000000000001},"min":{"1min":0.34200000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.68899999999999995}]},{"osd":5,"last update":"Wed Sep 18 07:24:09 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.434},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.47999999999999998}]},{"osd":6,"last update":"Wed Sep 18 07:24:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.50700000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56100000000000005}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.38800000000000001}]},{"osd":1,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.42499999999999999}]},{"osd":2,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.35099999999999998}]},{"osd":3,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.372},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.435}]},{"osd":4,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.58499999999999996}]},{"osd":6,"last update":"Wed Sep 18 07:24:32 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.52700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T07:24:53.168 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-18T07:24:53.374 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:53.804 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:53 smithi083 ceph-mon[23711]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:24:53.805 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:53 smithi083 ceph-mon[23711]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:54.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:53 smithi096 ceph-mon[25928]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:24:54.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:53 smithi096 ceph-mon[25928]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:55.143 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:24:55.145 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-09-18T07:24:55.749 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-09-18T07:24:54.544208+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49008,"kb_used_data":6384,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912208,"statfs":{"total":767960285184,"available":767910100992,"internally_reserved":0,"allocated":6537216,"data_stored":3572274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004083"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-18T07:24:00.623503+0000","last_change":"2024-09-18T07:24:00.623503+0000","last_active":"2024-09-18T07:24:00.623503+0000","last_peered":"2024-09-18T07:24:00.623503+0000","last_clean":"2024-09-18T07:24:00.623503+0000","last_became_active":"2024-09-18T07:24:00.623019+0000","last_became_peered":"2024-09-18T07:24:00.623019+0000","last_unstale":"2024-09-18T07:24:00.623503+0000","last_undegraded":"2024-09-18T07:24:00.623503+0000","last_fullsized":"2024-09-18T07:24:00.623503+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T07:21:59.049834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T07:21:59.049834+0000","last_clean_scrub_stamp":"2024-09-18T07:21:59.049834+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T10:34:03.872925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2662400,"data_stored":2644640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168591,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60299999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.504}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68899999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47999999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52100000000000002}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6120,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739032,"statfs":{"total":95995035648,"available":95988768768,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48499999999999999}]},{"osd":1,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.438},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.64200000000000002}]},{"osd":2,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.627},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.68899999999999995}]},{"osd":3,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.67300000000000004}]},{"osd":4,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.66400000000000003}]},{"osd":5,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.73899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73399999999999999}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:34 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.36399999999999999,"15min":0.34399999999999997},"min":{"1min":0.30499999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.35999999999999999,"15min":0.33800000000000002},"min":{"1min":0.23100000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.58099999999999996,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.29299999999999998}]},{"osd":2,"last update":"Wed Sep 18 07:24:00 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.39600000000000002,"15min":0.38900000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.44,"5min":0.41799999999999998,"15min":0.41399999999999998},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.33200000000000002}]},{"osd":3,"last update":"Wed Sep 18 07:24:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41199999999999998,"15min":0.40699999999999997},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.56799999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.46400000000000002,"15min":0.46100000000000002},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.41799999999999998}]},{"osd":4,"last update":"Wed Sep 18 07:24:45 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.58899999999999997,"15min":0.57599999999999996},"min":{"1min":0.47899999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.61199999999999999,"15min":0.60199999999999998},"min":{"1min":0.47899999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.57799999999999996}]},{"osd":5,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.501}]},{"osd":6,"last update":"Wed Sep 18 07:24:34 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.61399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63500000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Sep 18 07:24:37 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.33100000000000002,"15min":0.29499999999999998},"min":{"1min":0.221,"5min":0.22,"15min":0.22},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.378},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.34300000000000003,"15min":0.307},"min":{"1min":0.23200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.22500000000000001}]},{"osd":2,"last update":"Wed Sep 18 07:24:02 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.255,"5min":0.251,"15min":0.251},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.38600000000000001,"15min":0.378},"min":{"1min":0.27600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.29799999999999999}]},{"osd":3,"last update":"Wed Sep 18 07:24:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45600000000000002,"15min":0.45500000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.26500000000000001}]},{"osd":4,"last update":"Wed Sep 18 07:24:49 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.61299999999999999,"15min":0.60899999999999999},"min":{"1min":0.47699999999999998,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.61299999999999999,"15min":0.60899999999999999},"min":{"1min":0.45400000000000001,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.64800000000000002}]},{"osd":5,"last update":"Wed Sep 18 07:24:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.66000000000000003}]},{"osd":6,"last update":"Wed Sep 18 07:24:32 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.46700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.34699999999999998},"min":{"1min":0.27000000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.376,"15min":0.371},"min":{"1min":0.30299999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.308}]},{"osd":1,"last update":"Wed Sep 18 07:24:06 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38500000000000001,"15min":0.38300000000000001},"min":{"1min":0.29899999999999999,"5min":0.25,"15min":0.25},"max":{"1min":0.47999999999999998,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.39400000000000002,"15min":0.38900000000000001},"min":{"1min":0.29299999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.29699999999999999}]},{"osd":3,"last update":"Wed Sep 18 07:24:23 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.36299999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.58899999999999997,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.441,"15min":0.44},"min":{"1min":0.32900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.32800000000000001}]},{"osd":4,"last update":"Wed Sep 18 07:24:44 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.56799999999999995,"15min":0.55500000000000005},"min":{"1min":0.49099999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.497},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.59499999999999997,"15min":0.58399999999999996},"min":{"1min":0.499,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.53100000000000003}]},{"osd":5,"last update":"Wed Sep 18 07:24:12 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.44600000000000001}]},{"osd":6,"last update":"Wed Sep 18 07:24:31 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.46000000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.438}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:28 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40300000000000002,"15min":0.40400000000000003},"min":{"1min":0.27300000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.50600000000000001,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42899999999999999,"15min":0.42999999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.41899999999999998}]},{"osd":1,"last update":"Wed Sep 18 07:24:28 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.42099999999999999,"15min":0.41799999999999998},"min":{"1min":0.32700000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42899999999999999,"15min":0.42999999999999999},"min":{"1min":0.30499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.432}]},{"osd":2,"last update":"Wed Sep 18 07:24:28 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.38,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.438,"15min":0.441},"min":{"1min":0.315,"5min":0.312,"15min":0.312},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.497}]},{"osd":4,"last update":"Wed Sep 18 07:24:46 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.56799999999999995,"15min":0.55800000000000005},"min":{"1min":0.46500000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.56599999999999995,"15min":0.55900000000000005},"min":{"1min":0.44400000000000001,"5min":0.441,"15min":0.441},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.79600000000000004}]},{"osd":5,"last update":"Wed Sep 18 07:24:09 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.83199999999999996}]},{"osd":6,"last update":"Wed Sep 18 07:24:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.91400000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.89800000000000002}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5796,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739356,"statfs":{"total":95995035648,"available":95989100544,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.504,"15min":0.5},"min":{"1min":0.36899999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51700000000000002,"15min":0.51500000000000001},"min":{"1min":0.36599999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.69199999999999995}]},{"osd":1,"last update":"Wed Sep 18 07:24:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54100000000000004,"15min":0.54400000000000004},"min":{"1min":0.39200000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.503},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.52800000000000002,"15min":0.52200000000000002},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.63600000000000001}]},{"osd":2,"last update":"Wed Sep 18 07:24:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.53400000000000003,"15min":0.53500000000000003},"min":{"1min":0.41099999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.76700000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52100000000000002,"15min":0.51900000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.70999999999999996,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.73999999999999999}]},{"osd":3,"last update":"Wed Sep 18 07:24:48 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51600000000000001,"15min":0.51400000000000001},"min":{"1min":0.372,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.51700000000000002,"15min":0.51500000000000001},"min":{"1min":0.34200000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.60499999999999998}]},{"osd":5,"last update":"Wed Sep 18 07:24:09 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.80900000000000005}]},{"osd":6,"last update":"Wed Sep 18 07:24:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.753}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78000000000000003}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.38800000000000001}]},{"osd":1,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.42499999999999999}]},{"osd":2,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.35099999999999998}]},{"osd":3,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.372},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.435}]},{"osd":4,"last update":"Wed Sep 18 07:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.58499999999999996}]},{"osd":6,"last update":"Wed Sep 18 07:24:32 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.52700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T07:24:55.752 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-18T07:24:55.752 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-18T07:24:55.752 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-18T07:24:55.752 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-18T07:24:55.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:55 smithi083 ceph-mon[23711]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:55.969 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/mon.smithi083/config 2024-09-18T07:24:56.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:55 smithi096 ceph-mon[25928]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:56.755 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:56 smithi083 ceph-mon[23711]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:24:57.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:56 smithi096 ceph-mon[25928]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:24:57.785 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:24:57.785 INFO:teuthology.orchestra.run.smithi083.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-18T07:24:57.910 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:57 smithi083 ceph-mon[23711]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:57.916 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:57 smithi096 ceph-mon[25928]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:24:58.370 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-18T07:24:58.370 INFO:tasks.cephadm:Setup complete, yielding 2024-09-18T07:24:58.371 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-18T07:24:58.382 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:24:58.383 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-18T07:24:58.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:58 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/349297499' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-18T07:24:58.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:24:58.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:24:58.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:24:58.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:24:59.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:58 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/349297499' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-18T07:24:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:24:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:24:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:24:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:24:59.914 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:24:59 smithi083 ceph-mon[23711]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:00.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:24:59 smithi096 ceph-mon[25928]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:00.385 INFO:teuthology.orchestra.run.smithi083.stdout:Backend: cephadm 2024-09-18T07:25:00.385 INFO:teuthology.orchestra.run.smithi083.stdout:Available: Yes 2024-09-18T07:25:00.385 INFO:teuthology.orchestra.run.smithi083.stdout:Paused: No 2024-09-18T07:25:00.997 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-18T07:25:01.595 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:01 smithi083 ceph-mon[23711]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:01.595 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:01 smithi083 ceph-mon[23711]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:01.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:01 smithi096 ceph-mon[25928]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:01.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:01 smithi096 ceph-mon[25928]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (4m) 2m ago 5m 14.9M - ba2b418f427c 376c77ec4596 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (5m) 2m ago 5m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (5m) 67s ago 5m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (4m) 2m ago 5m 41.2M - 8.3.5 dad864ee21e9 1a3cff9a0185 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:9283 running (6m) 2m ago 6m 450M - 17.2.0 e1d6a67b021e 573994092cfb 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283 running (5m) 67s ago 5m 412M - 17.2.0 e1d6a67b021e 6e155b5f7eb8 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (6m) 2m ago 7m 41.9M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (4m) 67s ago 4m 39.9M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:25:02.921 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (5m) 2m ago 5m 17.4M - 1dbe0e931976 45810a37da0a 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (4m) 67s ago 4m 17.0M - 1dbe0e931976 3d6582ef7aff 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4m) 2m ago 4m 32.8M 3216M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3m) 2m ago 3m 34.1M 3216M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3m) 2m ago 3m 31.2M 3216M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2m) 2m ago 2m 10.9M 3216M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (2m) 67s ago 2m 32.8M 3984M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (2m) 67s ago 2m 32.9M 3984M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (97s) 67s ago 96s 33.5M 3984M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (72s) 67s ago 72s 12.0M 3984M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:25:02.922 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (4m) 2m ago 4m 53.5M - 514e6a882f6e 59f8176c5711 2024-09-18T07:25:03.535 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-18T07:25:03.833 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:03 smithi083 ceph-mon[23711]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:03.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:03 smithi096 ceph-mon[25928]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:04.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:04 smithi083 ceph-mon[23711]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:04.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:04 smithi096 ceph-mon[25928]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:05.520 INFO:teuthology.orchestra.run.smithi083.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-18T07:25:05.520 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-18T07:25:05.520 INFO:teuthology.orchestra.run.smithi083.stdout:crash 2/2 2m ago 6m * 2024-09-18T07:25:05.520 INFO:teuthology.orchestra.run.smithi083.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-18T07:25:05.520 INFO:teuthology.orchestra.run.smithi083.stdout:mgr 2/2 2m ago 6m count:2 2024-09-18T07:25:05.520 INFO:teuthology.orchestra.run.smithi083.stdout:mon 2/2 2m ago 5m smithi083:172.21.15.83=smithi083;smithi096:172.21.15.96=smithi096;count:2 2024-09-18T07:25:05.520 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-18T07:25:05.520 INFO:teuthology.orchestra.run.smithi083.stdout:osd 8 2m ago - 2024-09-18T07:25:05.520 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-18T07:25:05.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:05 smithi083 ceph-mon[23711]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:05.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:05 smithi096 ceph-mon[25928]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:06.117 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-18T07:25:06.721 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:06 smithi083 ceph-mon[23711]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:06.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:06 smithi096 ceph-mon[25928]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:07.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:07 smithi083 ceph-mon[23711]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:07.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:07 smithi096 ceph-mon[25928]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:07.985 INFO:teuthology.orchestra.run.smithi083.stdout:HOST ADDR LABELS STATUS 2024-09-18T07:25:07.985 INFO:teuthology.orchestra.run.smithi083.stdout:smithi083 172.21.15.83 2024-09-18T07:25:07.985 INFO:teuthology.orchestra.run.smithi083.stdout:smithi096 172.21.15.96 2024-09-18T07:25:07.986 INFO:teuthology.orchestra.run.smithi083.stdout:2 hosts in cluster 2024-09-18T07:25:08.663 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-18T07:25:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:09 smithi083 ceph-mon[23711]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:09 smithi083 ceph-mon[23711]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:09.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:09 smithi096 ceph-mon[25928]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:09 smithi096 ceph-mon[25928]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:10.581 INFO:teuthology.orchestra.run.smithi083.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-18T07:25:10.582 INFO:teuthology.orchestra.run.smithi083.stdout:smithi083 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62040148400BGN 400G LVM detected, locked 2024-09-18T07:25:10.582 INFO:teuthology.orchestra.run.smithi083.stdout:smithi096 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400NH400BGN 400G LVM detected, locked 2024-09-18T07:25:11.376 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-18T07:25:11.388 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:25:11.388 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-18T07:25:11.414 INFO:teuthology.orchestra.run.smithi083.stderr:+ systemctl stop nfs-server 2024-09-18T07:25:11.422 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi096.front.sepia.ceph.com 2024-09-18T07:25:11.422 DEBUG:teuthology.orchestra.run.smithi096:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-18T07:25:11.452 INFO:teuthology.orchestra.run.smithi096.stderr:+ systemctl stop nfs-server 2024-09-18T07:25:11.461 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-18T07:25:11.470 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:25:11.470 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-18T07:25:11.617 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:11 smithi083 ceph-mon[23711]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:11.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:11 smithi083 ceph-mon[23711]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:11 smithi096 ceph-mon[25928]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:11.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:11 smithi096 ceph-mon[25928]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:13.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:13 smithi083 ceph-mon[23711]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:13.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:13 smithi096 ceph-mon[25928]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:14.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:14 smithi083 ceph-mon[23711]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:14.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:14 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-18T07:25:14.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:14 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-18T07:25:14.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:14 smithi083 ceph-mon[23711]: osdmap e61: 8 total, 8 up, 8 in 2024-09-18T07:25:14.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:14 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-18T07:25:14.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:14 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083[23707]: 2024-09-18T07:25:14.675+0000 7fbc8a739700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-18T07:25:14.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:14 smithi096 ceph-mon[25928]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:14.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:14 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-18T07:25:14.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:14 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-18T07:25:14.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:14 smithi096 ceph-mon[25928]: osdmap e61: 8 total, 8 up, 8 in 2024-09-18T07:25:14.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:14 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-18T07:25:15.501 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-18T07:25:15.510 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-18T07:25:15.510 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: pgmap v214: 33 pgs: 3 creating+peering, 1 active+clean, 29 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: osdmap e62: 8 total, 8 up, 8 in 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: osdmap e63: 8 total, 8 up, 8 in 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: fsmap foofs:0 2024-09-18T07:25:15.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: Saving service mds.foofs spec with placement count:2 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.kjbbry", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.kjbbry", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:15.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:15 smithi083 ceph-mon[23711]: Deploying daemon mds.foofs.smithi083.kjbbry on smithi083 2024-09-18T07:25:16.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: pgmap v214: 33 pgs: 3 creating+peering, 1 active+clean, 29 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:16.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-18T07:25:16.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: osdmap e62: 8 total, 8 up, 8 in 2024-09-18T07:25:16.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-18T07:25:16.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-18T07:25:16.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: osdmap e63: 8 total, 8 up, 8 in 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: fsmap foofs:0 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: Saving service mds.foofs spec with placement count:2 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.kjbbry", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.kjbbry", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:16.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:15 smithi096 ceph-mon[25928]: Deploying daemon mds.foofs.smithi083.kjbbry on smithi083 2024-09-18T07:25:16.826 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:16 smithi083 ceph-mon[23711]: osdmap e64: 8 total, 8 up, 8 in 2024-09-18T07:25:17.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:16 smithi096 ceph-mon[25928]: osdmap e64: 8 total, 8 up, 8 in 2024-09-18T07:25:17.814 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:17 smithi083 ceph-mon[23711]: pgmap v218: 65 pgs: 2 creating+activating, 3 creating+peering, 8 active+clean, 52 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:17.814 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:17 smithi083 ceph-mon[23711]: osdmap e65: 8 total, 8 up, 8 in 2024-09-18T07:25:18.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:17 smithi096 ceph-mon[25928]: pgmap v218: 65 pgs: 2 creating+activating, 3 creating+peering, 8 active+clean, 52 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:18.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:17 smithi096 ceph-mon[25928]: osdmap e65: 8 total, 8 up, 8 in 2024-09-18T07:25:18.214 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:25:18.215 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-18T07:18:45.888678Z", "last_refresh": "2024-09-18T07:22:22.407511Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-18T07:18:40.384022Z", "last_refresh": "2024-09-18T07:22:22.407577Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-18T07:18:43.135860Z", "last_refresh": "2024-09-18T07:22:22.407639Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-18T07:25:14.699088Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-18T07:25:14.694022Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-18T07:18:38.506217Z", "last_refresh": "2024-09-18T07:22:22.407439Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi096:172.21.15.96=smithi096"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-18T07:19:27.514426Z", "last_refresh": "2024-09-18T07:22:22.407298Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-18T07:18:44.228178Z", "last_refresh": "2024-09-18T07:22:22.407700Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-18T07:22:22.407838Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-18T07:18:42.006767Z", "last_refresh": "2024-09-18T07:22:22.407761Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-18T07:25:18.805 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-18T07:25:19.806 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-18T07:25:20.110 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:19 smithi083 ceph-mon[23711]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:25:20.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:19 smithi083 ceph-mon[23711]: pgmap v220: 65 pgs: 2 creating+activating, 3 creating+peering, 40 active+clean, 20 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:20.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:19 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:20.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:19 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi096.bzilgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-18T07:25:20.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:19 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi096.bzilgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-18T07:25:20.111 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:19 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:19 smithi096 ceph-mon[25928]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:25:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:19 smithi096 ceph-mon[25928]: pgmap v220: 65 pgs: 2 creating+activating, 3 creating+peering, 40 active+clean, 20 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:19 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:19 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi096.bzilgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-18T07:25:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:19 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi096.bzilgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-18T07:25:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:19 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: Deploying daemon mds.foofs.smithi096.bzilgv on smithi096 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: mds.? [v2:172.21.15.83:6834/2180084897,v1:172.21.15.83:6835/2180084897] up:boot 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: daemon mds.foofs.smithi083.kjbbry assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: Cluster is now healthy 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: fsmap foofs:0 1 up:standby 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"}]: dispatch 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:creating} 2024-09-18T07:25:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:20 smithi096 ceph-mon[25928]: daemon mds.foofs.smithi083.kjbbry is now active in filesystem foofs as rank 0 2024-09-18T07:25:21.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: Deploying daemon mds.foofs.smithi096.bzilgv on smithi096 2024-09-18T07:25:21.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: mds.? [v2:172.21.15.83:6834/2180084897,v1:172.21.15.83:6835/2180084897] up:boot 2024-09-18T07:25:21.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: daemon mds.foofs.smithi083.kjbbry assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-18T07:25:21.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-18T07:25:21.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-18T07:25:21.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: Cluster is now healthy 2024-09-18T07:25:21.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: fsmap foofs:0 1 up:standby 2024-09-18T07:25:21.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"}]: dispatch 2024-09-18T07:25:21.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:creating} 2024-09-18T07:25:21.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:20 smithi083 ceph-mon[23711]: daemon mds.foofs.smithi083.kjbbry is now active in filesystem foofs as rank 0 2024-09-18T07:25:21.854 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:25:21.854 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-18T07:18:45.888678Z", "last_refresh": "2024-09-18T07:22:22.407511Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-18T07:18:40.384022Z", "last_refresh": "2024-09-18T07:22:22.407577Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-18T07:18:43.135860Z", "last_refresh": "2024-09-18T07:22:22.407639Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-18T07:25:14.699088Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-18T07:25:14.694022Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-18T07:18:38.506217Z", "last_refresh": "2024-09-18T07:22:22.407439Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi096:172.21.15.96=smithi096"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-18T07:19:27.514426Z", "last_refresh": "2024-09-18T07:22:22.407298Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-18T07:18:44.228178Z", "last_refresh": "2024-09-18T07:22:22.407700Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-18T07:22:22.407838Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-18T07:18:42.006767Z", "last_refresh": "2024-09-18T07:22:22.407761Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-18T07:25:21.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:21 smithi096 ceph-mon[25928]: pgmap v221: 65 pgs: 2 creating+activating, 63 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:21.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:21 smithi096 ceph-mon[25928]: mds.? [v2:172.21.15.83:6834/2180084897,v1:172.21.15.83:6835/2180084897] up:active 2024-09-18T07:25:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:21 smithi096 ceph-mon[25928]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 2024-09-18T07:25:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:21 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:21 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:21 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:21 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:22.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:21 smithi083 ceph-mon[23711]: pgmap v221: 65 pgs: 2 creating+activating, 63 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:22.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:21 smithi083 ceph-mon[23711]: mds.? [v2:172.21.15.83:6834/2180084897,v1:172.21.15.83:6835/2180084897] up:active 2024-09-18T07:25:22.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:21 smithi083 ceph-mon[23711]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 2024-09-18T07:25:22.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:21 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:22.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:21 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:22.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:21 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:22.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:21 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:22.477 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-18T07:25:22.887 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:22 smithi096 ceph-mon[25928]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:boot 2024-09-18T07:25:22.887 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:22 smithi096 ceph-mon[25928]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:25:22.887 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:22 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"}]: dispatch 2024-09-18T07:25:22.887 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:22 smithi096 ceph-mon[25928]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:25:23.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:22 smithi083 ceph-mon[23711]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:boot 2024-09-18T07:25:23.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:22 smithi083 ceph-mon[23711]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:25:23.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:22 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"}]: dispatch 2024-09-18T07:25:23.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:22 smithi083 ceph-mon[23711]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:25:23.478 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-18T07:25:23.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:23 smithi083 ceph-mon[23711]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:25:23.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:23 smithi083 ceph-mon[23711]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 260 B/s wr, 0 op/s 2024-09-18T07:25:23.971 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:23 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:23 smithi096 ceph-mon[25928]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:25:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:23 smithi096 ceph-mon[25928]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 260 B/s wr, 0 op/s 2024-09-18T07:25:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:23 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:25.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:25 smithi096 ceph-mon[25928]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-09-18T07:25:25.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:25 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:25.859 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:25 smithi083 ceph-mon[23711]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-09-18T07:25:25.860 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:25 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:26.463 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:25:26.463 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-18T07:18:45.888678Z", "last_refresh": "2024-09-18T07:22:22.407511Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-18T07:18:40.384022Z", "last_refresh": "2024-09-18T07:22:22.407577Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-18T07:18:43.135860Z", "last_refresh": "2024-09-18T07:22:22.407639Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-18T07:25:14.699088Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-18T07:25:14.694022Z", "last_refresh": "2024-09-18T07:25:23.417891Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-18T07:18:38.506217Z", "last_refresh": "2024-09-18T07:22:22.407439Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi096:172.21.15.96=smithi096"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-18T07:19:27.514426Z", "last_refresh": "2024-09-18T07:22:22.407298Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-18T07:18:44.228178Z", "last_refresh": "2024-09-18T07:22:22.407700Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-18T07:22:22.407838Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-18T07:18:42.006767Z", "last_refresh": "2024-09-18T07:22:22.407761Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-18T07:25:27.408 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-18T07:25:27.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:27 smithi083 ceph-mon[23711]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:25:27.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:27 smithi083 ceph-mon[23711]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-18T07:25:27.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:27 smithi096 ceph-mon[25928]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:25:27.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:27 smithi096 ceph-mon[25928]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-18T07:25:28.409 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-18T07:25:29.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:29.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:29.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:29.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:29.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:29.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:29 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:29.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:29 smithi083 ceph-mon[23711]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-18T07:25:29.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:29.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:29 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:29.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:29 smithi096 ceph-mon[25928]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-18T07:25:30.384 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:25:30.385 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-18T07:18:45.888678Z", "last_refresh": "2024-09-18T07:25:28.342588Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-18T07:18:40.384022Z", "last_refresh": "2024-09-18T07:25:23.417290Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-18T07:18:43.135860Z", "last_refresh": "2024-09-18T07:25:28.342713Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-18T07:25:14.699088Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-18T07:25:14.694022Z", "last_refresh": "2024-09-18T07:25:23.417891Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-18T07:18:38.506217Z", "last_refresh": "2024-09-18T07:25:23.417433Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi096:172.21.15.96=smithi096"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-18T07:19:27.514426Z", "last_refresh": "2024-09-18T07:25:23.417504Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-18T07:18:44.228178Z", "last_refresh": "2024-09-18T07:25:23.417569Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-18T07:25:23.417633Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-18T07:18:42.006767Z", "last_refresh": "2024-09-18T07:25:28.342851Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-18T07:25:30.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:30 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:30 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:31.044 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-18T07:25:31.044 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-18T07:25:31.053 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:25:31.053 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-18T07:25:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:31 smithi083 ceph-mon[23711]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:25:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:31 smithi083 ceph-mon[23711]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-18T07:25:31.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:31 smithi096 ceph-mon[25928]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:25:31.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:31 smithi096 ceph-mon[25928]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-18T07:25:33.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:33 smithi083 ceph-mon[23711]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-18T07:25:33.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:33 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-18T07:25:33.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:33 smithi096 ceph-mon[25928]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-18T07:25:33.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:33 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-18T07:25:34.627 INFO:teuthology.orchestra.run.smithi083.stdout:NFS Cluster Created Successfully 2024-09-18T07:25:34.845 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:34 smithi083 ceph-mon[23711]: from='client.14520 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:34.845 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:34 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-18T07:25:34.845 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:34 smithi083 ceph-mon[23711]: osdmap e66: 8 total, 8 up, 8 in 2024-09-18T07:25:34.845 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:34 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-18T07:25:34.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:34 smithi096 ceph-mon[25928]: from='client.14520 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:34 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-18T07:25:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:34 smithi096 ceph-mon[25928]: osdmap e66: 8 total, 8 up, 8 in 2024-09-18T07:25:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:34 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-18T07:25:35.308 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-18T07:25:35.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: pgmap v229: 97 pgs: 16 creating+peering, 16 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-18T07:25:35.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-18T07:25:35.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: osdmap e67: 8 total, 8 up, 8 in 2024-09-18T07:25:35.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: Saving service nfs.foo spec with placement count:2 2024-09-18T07:25:35.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: Creating key for client.nfs.foo.0.0.smithi083.jpbwtk 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:35.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-18T07:25:35.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-18T07:25:35.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-18T07:25:35.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-18T07:25:35.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:35 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:35.848 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: pgmap v229: 97 pgs: 16 creating+peering, 16 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: osdmap e67: 8 total, 8 up, 8 in 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: Saving service nfs.foo spec with placement count:2 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: Creating key for client.nfs.foo.0.0.smithi083.jpbwtk 2024-09-18T07:25:35.849 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-18T07:25:35.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:35 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:36.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:36 smithi083 ceph-mon[23711]: Rados config object exists: conf-nfs.foo 2024-09-18T07:25:36.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:36 smithi083 ceph-mon[23711]: Creating key for client.nfs.foo.0.0.smithi083.jpbwtk-rgw 2024-09-18T07:25:36.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:36 smithi083 ceph-mon[23711]: Deploying daemon nfs.foo.0.0.smithi083.jpbwtk on smithi083 2024-09-18T07:25:36.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:36 smithi083 ceph-mon[23711]: osdmap e68: 8 total, 8 up, 8 in 2024-09-18T07:25:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:36 smithi096 ceph-mon[25928]: Rados config object exists: conf-nfs.foo 2024-09-18T07:25:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:36 smithi096 ceph-mon[25928]: Creating key for client.nfs.foo.0.0.smithi083.jpbwtk-rgw 2024-09-18T07:25:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:36 smithi096 ceph-mon[25928]: Deploying daemon nfs.foo.0.0.smithi083.jpbwtk on smithi083 2024-09-18T07:25:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:36 smithi096 ceph-mon[25928]: osdmap e68: 8 total, 8 up, 8 in 2024-09-18T07:25:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:37 smithi083 ceph-mon[23711]: pgmap v232: 97 pgs: 16 creating+peering, 11 unknown, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:37.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:37 smithi083 ceph-mon[23711]: mgrmap e20: smithi083.vchvnu(active, since 5m), standbys: smithi096.caifnv 2024-09-18T07:25:37.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:37 smithi096 ceph-mon[25928]: pgmap v232: 97 pgs: 16 creating+peering, 11 unknown, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-18T07:25:37.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:37 smithi096 ceph-mon[25928]: mgrmap e20: smithi083.vchvnu(active, since 5m), standbys: smithi096.caifnv 2024-09-18T07:25:38.719 INFO:teuthology.orchestra.run.smithi083.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-18T07:25:38.719 INFO:teuthology.orchestra.run.smithi083.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-18T07:25:38.719 INFO:teuthology.orchestra.run.smithi083.stderr:Error EINVAL: invalid command 2024-09-18T07:25:39.073 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:25:39.074 INFO:teuthology.orchestra.run.smithi083.stdout: "bind": "/fake", 2024-09-18T07:25:39.074 INFO:teuthology.orchestra.run.smithi083.stdout: "fs": "foofs", 2024-09-18T07:25:39.074 INFO:teuthology.orchestra.run.smithi083.stdout: "path": "/", 2024-09-18T07:25:39.074 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster": "foo", 2024-09-18T07:25:39.074 INFO:teuthology.orchestra.run.smithi083.stdout: "mode": "RW" 2024-09-18T07:25:39.074 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:25:39.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:39 smithi083 ceph-mon[23711]: pgmap v233: 97 pgs: 16 creating+peering, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-18T07:25:39.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:39.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-18T07:25:39.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-18T07:25:39.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-18T07:25:39.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-18T07:25:39.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:39.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-18T07:25:39.748 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:39 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-18T07:25:39.770 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-18T07:25:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:39 smithi096 ceph-mon[25928]: pgmap v233: 97 pgs: 16 creating+peering, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-18T07:25:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-18T07:25:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-18T07:25:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-18T07:25:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-18T07:25:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-18T07:25:39.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:39 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-18T07:25:40.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:40 smithi083 ceph-mon[23711]: Creating key for client.nfs.foo.1.0.smithi096.vijuth 2024-09-18T07:25:40.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:40 smithi083 ceph-mon[23711]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-18T07:25:40.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:40 smithi083 ceph-mon[23711]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:40.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:40 smithi096 ceph-mon[25928]: Creating key for client.nfs.foo.1.0.smithi096.vijuth 2024-09-18T07:25:40.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:40 smithi096 ceph-mon[25928]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-18T07:25:40.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:40 smithi096 ceph-mon[25928]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:41.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:41 smithi083 ceph-mon[23711]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 147 B/s rd, 294 B/s wr, 0 op/s 2024-09-18T07:25:41.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:41 smithi096 ceph-mon[25928]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 147 B/s rd, 294 B/s wr, 0 op/s 2024-09-18T07:25:42.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:42 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-18T07:25:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:42 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-18T07:25:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:42 smithi083 ceph-mon[23711]: Rados config object exists: conf-nfs.foo 2024-09-18T07:25:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:42 smithi083 ceph-mon[23711]: Creating key for client.nfs.foo.1.0.smithi096.vijuth-rgw 2024-09-18T07:25:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:42 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-18T07:25:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:42 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-18T07:25:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:42 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:42.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:42 smithi083 ceph-mon[23711]: Deploying daemon nfs.foo.1.0.smithi096.vijuth on smithi096 2024-09-18T07:25:42.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:42 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-18T07:25:42.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:42 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-18T07:25:42.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:42 smithi096 ceph-mon[25928]: Rados config object exists: conf-nfs.foo 2024-09-18T07:25:42.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:42 smithi096 ceph-mon[25928]: Creating key for client.nfs.foo.1.0.smithi096.vijuth-rgw 2024-09-18T07:25:42.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:42 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-18T07:25:42.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:42 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-18T07:25:42.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:42 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:42.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:42 smithi096 ceph-mon[25928]: Deploying daemon nfs.foo.1.0.smithi096.vijuth on smithi096 2024-09-18T07:25:44.004 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:43 smithi096 ceph-mon[25928]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:44.004 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:43 smithi096 ceph-mon[25928]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-09-18T07:25:44.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:43 smithi083 ceph-mon[23711]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:44.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:43 smithi083 ceph-mon[23711]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-09-18T07:25:44.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:44.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:44.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:44.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:44.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:44.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-18T07:25:44.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2024-09-18T07:25:44.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-18T07:25:44.870 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-18T07:25:44.870 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:25:44.870 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:25:44.870 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-18T07:25:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]: dispatch 2024-09-18T07:25:44.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-18T07:25:44.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-18T07:25:44.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:25:44.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:25:44.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:46.107 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:45 smithi096 ceph-mon[25928]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:46.108 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:45 smithi096 ceph-mon[25928]: pgmap v236: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-18T07:25:46.108 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:45 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-18T07:25:46.108 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:45 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2024-09-18T07:25:46.108 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:45 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-18T07:25:46.108 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:45 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-18T07:25:46.108 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:45 smithi096 ceph-mon[25928]: osdmap e69: 8 total, 8 up, 8 in 2024-09-18T07:25:46.117 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:45 smithi083 ceph-mon[23711]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:46.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:45 smithi083 ceph-mon[23711]: pgmap v236: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-18T07:25:46.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-18T07:25:46.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 5]}]': finished 2024-09-18T07:25:46.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-18T07:25:46.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:45 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-18T07:25:46.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:45 smithi083 ceph-mon[23711]: osdmap e69: 8 total, 8 up, 8 in 2024-09-18T07:25:46.835 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:46 smithi096 ceph-mon[25928]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:46.835 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:46 smithi096 ceph-mon[25928]: osdmap e70: 8 total, 8 up, 8 in 2024-09-18T07:25:46.835 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:46 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:47.045 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:46 smithi083 ceph-mon[23711]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:47.045 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:46 smithi083 ceph-mon[23711]: osdmap e70: 8 total, 8 up, 8 in 2024-09-18T07:25:47.045 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:46 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:47.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:47 smithi083 ceph-mon[23711]: pgmap v239: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-09-18T07:25:48.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:47 smithi096 ceph-mon[25928]: pgmap v239: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-09-18T07:25:48.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:48 smithi083 ceph-mon[23711]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:49.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:48 smithi096 ceph-mon[25928]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:49.647 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo ?:2049 2/2 3s ago 15s count:2 2024-09-18T07:25:49.915 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:49.915 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:49.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:49.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:49.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:49.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:49.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:49.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:49.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:49.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:49.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:49 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:49 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:25:50.373 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-18T07:25:50.383 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:25:50.384 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-18T07:25:50.410 INFO:teuthology.orchestra.run.smithi083.stderr:+ mkdir /mnt/foo 2024-09-18T07:25:50.423 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-18T07:25:50.488 INFO:teuthology.orchestra.run.smithi083.stderr:++ hostname 2024-09-18T07:25:50.488 INFO:teuthology.orchestra.run.smithi083.stderr:+ mount -t nfs smithi083:/fake /mnt/foo -o sync 2024-09-18T07:25:50.908 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-18T07:25:50.972 INFO:teuthology.orchestra.run.smithi083.stderr:+ echo test 2024-09-18T07:25:50.989 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:50 smithi083 ceph-mon[23711]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:50.990 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-18T07:25:51.053 INFO:teuthology.orchestra.run.smithi083.stderr:+ sync 2024-09-18T07:25:51.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:50 smithi096 ceph-mon[25928]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:25:51.249 INFO:teuthology.run_tasks:Running task parallel... 2024-09-18T07:25:51.262 INFO:teuthology.task.parallel:starting parallel... 2024-09-18T07:25:51.262 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-18T07:25:51.263 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-18T07:25:51.263 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:25:51.263 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-18T07:25:51.264 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-18T07:25:51.264 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-18T07:25:51.265 INFO:teuthology.task.exec:Executing custom commands... 2024-09-18T07:25:51.265 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:25:51.265 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-18T07:25:51.366 INFO:teuthology.orchestra.run.smithi083.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-18T07:25:51.367 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:25:51.367 INFO:teuthology.orchestra.run.smithi083.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-18T07:25:51.367 INFO:teuthology.orchestra.run.smithi083.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-18T07:25:51.468 INFO:teuthology.orchestra.run.smithi083.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-18T07:25:51.468 INFO:teuthology.orchestra.run.smithi083.stdout:releasing clients 2024-09-18T07:25:51.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:51 smithi083 ceph-mon[23711]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 71 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:52.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:51 smithi096 ceph-mon[25928]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 71 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:52.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 116 26.54 MB/sec warmup 1 sec latency 36.601 ms 2024-09-18T07:25:53.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 249 23.61 MB/sec warmup 2 sec latency 30.674 ms 2024-09-18T07:25:53.940 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:53 smithi083 ceph-mon[23711]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.5 KiB/s wr, 4 op/s; 71 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:54.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:53 smithi096 ceph-mon[25928]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.5 KiB/s wr, 4 op/s; 71 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:54.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 403 23.74 MB/sec warmup 3 sec latency 32.835 ms 2024-09-18T07:25:55.011 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-18T07:25:55.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 527 22.93 MB/sec warmup 4 sec latency 47.797 ms 2024-09-18T07:25:55.843 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:55 smithi083 ceph-mon[23711]: pgmap v243: 97 pgs: 97 active+clean; 42 MiB data, 78 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 4.4 MiB/s wr, 140 op/s; 59 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:56.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:55 smithi096 ceph-mon[25928]: pgmap v243: 97 pgs: 97 active+clean; 42 MiB data, 78 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 4.4 MiB/s wr, 140 op/s; 59 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:56.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 854 21.59 MB/sec warmup 5 sec latency 30.475 ms 2024-09-18T07:25:57.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 1387 21.20 MB/sec warmup 6 sec latency 31.895 ms 2024-09-18T07:25:57.859 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:57 smithi083 ceph-mon[23711]: pgmap v244: 97 pgs: 97 active+clean; 51 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 4.9 MiB/s wr, 144 op/s; 54 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:58.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:57 smithi096 ceph-mon[25928]: pgmap v244: 97 pgs: 97 active+clean; 51 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 4.9 MiB/s wr, 144 op/s; 54 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:58.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 2296 19.64 MB/sec warmup 7 sec latency 23.906 ms 2024-09-18T07:25:58.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:25:58 smithi083 ceph-mon[23711]: pgmap v245: 97 pgs: 97 active+clean; 90 MiB data, 176 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 8.0 MiB/s wr, 413 op/s; 47 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:25:58 smithi096 ceph-mon[25928]: pgmap v245: 97 pgs: 97 active+clean; 90 MiB data, 176 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 8.0 MiB/s wr, 413 op/s; 47 B/s, 0 keys/s, 0 objects/s recovering 2024-09-18T07:25:59.396 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-18T07:25:59.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 2895 20.45 MB/sec warmup 8 sec latency 22.522 ms 2024-09-18T07:26:00.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 3571 21.37 MB/sec warmup 9 sec latency 43.348 ms 2024-09-18T07:26:01.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 3882 19.91 MB/sec warmup 10 sec latency 31.180 ms 2024-09-18T07:26:01.654 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:01 smithi083 ceph-mon[23711]: pgmap v246: 97 pgs: 97 active+clean; 113 MiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 10 MiB/s wr, 487 op/s; 46 B/s, 0 objects/s recovering 2024-09-18T07:26:01.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:01 smithi096 ceph-mon[25928]: pgmap v246: 97 pgs: 97 active+clean; 113 MiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 10 MiB/s wr, 487 op/s; 46 B/s, 0 objects/s recovering 2024-09-18T07:26:02.468 INFO:teuthology.orchestra.run.smithi083.stdout: 5 4404 18.48 MB/sec warmup 11 sec latency 50.593 ms 2024-09-18T07:26:03.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 5121 18.85 MB/sec warmup 12 sec latency 36.658 ms 2024-09-18T07:26:03.670 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:03 smithi083 ceph-mon[23711]: pgmap v247: 97 pgs: 97 active+clean; 114 MiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 11 MiB/s wr, 498 op/s 2024-09-18T07:26:03.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:03 smithi096 ceph-mon[25928]: pgmap v247: 97 pgs: 97 active+clean; 114 MiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 11 MiB/s wr, 498 op/s 2024-09-18T07:26:04.397 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph mgr module enable nfs --force' 2024-09-18T07:26:04.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 5824 17.79 MB/sec warmup 13 sec latency 38.016 ms 2024-09-18T07:26:05.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 6077 17.09 MB/sec warmup 14 sec latency 71.349 ms 2024-09-18T07:26:05.697 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:05 smithi083 ceph-mon[23711]: pgmap v248: 97 pgs: 97 active+clean; 130 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 14 MiB/s wr, 810 op/s 2024-09-18T07:26:05.869 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:05 smithi096 ceph-mon[25928]: pgmap v248: 97 pgs: 97 active+clean; 130 MiB data, 248 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 14 MiB/s wr, 810 op/s 2024-09-18T07:26:06.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 6634 17.64 MB/sec warmup 15 sec latency 38.769 ms 2024-09-18T07:26:07.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 7158 18.03 MB/sec warmup 16 sec latency 47.252 ms 2024-09-18T07:26:07.630 INFO:teuthology.orchestra.run.smithi083.stderr:module 'nfs' is already enabled 2024-09-18T07:26:07.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:07 smithi083 ceph-mon[23711]: pgmap v249: 97 pgs: 97 active+clean; 130 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 11 MiB/s wr, 713 op/s 2024-09-18T07:26:07.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:07 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/871376137' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-18T07:26:08.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:07 smithi096 ceph-mon[25928]: pgmap v249: 97 pgs: 97 active+clean; 130 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 11 MiB/s wr, 713 op/s 2024-09-18T07:26:08.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:07 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/871376137' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-18T07:26:08.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 7414 17.34 MB/sec warmup 17 sec latency 41.377 ms 2024-09-18T07:26:08.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:08 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/871376137' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-18T07:26:08.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:08 smithi083 ceph-mon[23711]: mgrmap e21: smithi083.vchvnu(active, since 6m), standbys: smithi096.caifnv 2024-09-18T07:26:08.945 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-18T07:26:09.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:08 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/871376137' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-18T07:26:09.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:08 smithi096 ceph-mon[25928]: mgrmap e21: smithi083.vchvnu(active, since 6m), standbys: smithi096.caifnv 2024-09-18T07:26:09.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 7810 16.51 MB/sec warmup 18 sec latency 64.041 ms 2024-09-18T07:26:09.781 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:09 smithi083 ceph-mon[23711]: pgmap v250: 97 pgs: 97 active+clean; 133 MiB data, 308 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 12 MiB/s wr, 929 op/s 2024-09-18T07:26:10.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:09 smithi096 ceph-mon[25928]: pgmap v250: 97 pgs: 97 active+clean; 133 MiB data, 308 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 12 MiB/s wr, 929 op/s 2024-09-18T07:26:10.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 8084 15.91 MB/sec warmup 19 sec latency 77.864 ms 2024-09-18T07:26:11.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 8415 15.62 MB/sec warmup 20 sec latency 74.243 ms 2024-09-18T07:26:11.607 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:11 smithi083 ceph-mon[23711]: pgmap v251: 97 pgs: 97 active+clean; 139 MiB data, 336 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.0 MiB/s wr, 682 op/s 2024-09-18T07:26:11.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:11 smithi096 ceph-mon[25928]: pgmap v251: 97 pgs: 97 active+clean; 139 MiB data, 336 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.0 MiB/s wr, 682 op/s 2024-09-18T07:26:11.929 INFO:teuthology.orchestra.run.smithi083.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e 2024-09-18T07:26:12.469 INFO:teuthology.orchestra.run.smithi083.stdout: 5 8778 15.34 MB/sec warmup 21 sec latency 79.627 ms 2024-09-18T07:26:13.009 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-18T07:26:13.010 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:26:13.010 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-18T07:26:13.140 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:13.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e 2024-09-18T07:26:13.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:13.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:26:13.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:26:13.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:26:13.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:13.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:13.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e 2024-09-18T07:26:13.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:12 smithi083 ceph-mon[23711]: pgmap v252: 97 pgs: 97 active+clean; 138 MiB data, 336 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.7 MiB/s wr, 612 op/s 2024-09-18T07:26:13.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:13.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e 2024-09-18T07:26:13.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:13.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:26:13.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:26:13.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:26:13.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:13.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:13.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e 2024-09-18T07:26:13.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:12 smithi096 ceph-mon[25928]: pgmap v252: 97 pgs: 97 active+clean; 138 MiB data, 336 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.7 MiB/s wr, 612 op/s 2024-09-18T07:26:13.472 INFO:teuthology.orchestra.run.smithi083.stdout: 5 9253 14.82 MB/sec warmup 22 sec latency 60.729 ms 2024-09-18T07:26:14.472 INFO:teuthology.orchestra.run.smithi083.stdout: 5 9581 14.49 MB/sec warmup 23 sec latency 41.254 ms 2024-09-18T07:26:15.472 INFO:teuthology.orchestra.run.smithi083.stdout: 5 10192 15.05 MB/sec warmup 24 sec latency 44.469 ms 2024-09-18T07:26:15.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:15 smithi083 ceph-mon[23711]: pgmap v253: 97 pgs: 97 active+clean; 136 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 825 op/s 2024-09-18T07:26:15.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:15 smithi096 ceph-mon[25928]: pgmap v253: 97 pgs: 97 active+clean; 136 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 825 op/s 2024-09-18T07:26:16.420 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:26:16.472 INFO:teuthology.orchestra.run.smithi083.stdout: 5 10705 15.28 MB/sec warmup 25 sec latency 45.239 ms 2024-09-18T07:26:17.355 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:26:17.355 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (5m) 28s ago 6m 24.6M - ba2b418f427c 376c77ec4596 2024-09-18T07:26:17.355 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (6m) 28s ago 6m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:26:17.355 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (6m) 30s ago 6m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:26:17.355 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (5m) 28s ago 6m 49.7M - 8.3.5 dad864ee21e9 1a3cff9a0185 2024-09-18T07:26:17.355 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (59s) 28s ago 58s 15.8M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:26:17.355 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (56s) 30s ago 55s 13.4M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:26:17.355 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:9283 running (8m) 28s ago 8m 462M - 17.2.0 e1d6a67b021e 573994092cfb 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283 running (6m) 30s ago 6m 413M - 17.2.0 e1d6a67b021e 6e155b5f7eb8 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (8m) 28s ago 8m 46.9M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (6m) 30s ago 6m 37.4M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (39s) 28s ago 38s 54.3M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (33s) 30s ago 33s 50.8M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (6m) 28s ago 6m 17.3M - 1dbe0e931976 45810a37da0a 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (6m) 30s ago 6m 16.9M - 1dbe0e931976 3d6582ef7aff 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 28s ago 5m 42.7M 3216M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4m) 28s ago 4m 42.6M 3216M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 28s ago 4m 40.3M 3216M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 28s ago 4m 40.7M 3216M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (3m) 30s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (3m) 30s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:26:17.356 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (2m) 30s ago 2m 43.7M 3984M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:26:17.357 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (2m) 30s ago 2m 40.9M 3984M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:26:17.357 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (5m) 28s ago 5m 62.3M - 514e6a882f6e 59f8176c5711 2024-09-18T07:26:17.472 INFO:teuthology.orchestra.run.smithi083.stdout: 5 11000 15.03 MB/sec warmup 26 sec latency 39.591 ms 2024-09-18T07:26:17.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:17 smithi083 ceph-mon[23711]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:17.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:17 smithi083 ceph-mon[23711]: pgmap v254: 97 pgs: 97 active+clean; 138 MiB data, 406 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 522 op/s 2024-09-18T07:26:17.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:17 smithi096 ceph-mon[25928]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:17 smithi096 ceph-mon[25928]: pgmap v254: 97 pgs: 97 active+clean; 138 MiB data, 406 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 522 op/s 2024-09-18T07:26:17.974 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:26:17.975 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:26:17.976 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:26:17.976 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:26:17.976 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-18T07:26:17.976 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:26:17.976 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:26:18.472 INFO:teuthology.orchestra.run.smithi083.stdout: 5 11319 14.54 MB/sec warmup 27 sec latency 51.073 ms 2024-09-18T07:26:18.504 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:26:18.505 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e", 2024-09-18T07:26:18.505 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:26:18.505 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-09-18T07:26:18.505 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "", 2024-09-18T07:26:18.505 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e image" 2024-09-18T07:26:18.505 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:26:18.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:18 smithi083 ceph-mon[23711]: from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:18.649 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:18 smithi083 ceph-mon[23711]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:18.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:18 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1137155036' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:26:19.011 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:26:19.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:18 smithi096 ceph-mon[25928]: from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:19.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:18 smithi096 ceph-mon[25928]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:19.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:18 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1137155036' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:26:19.472 INFO:teuthology.orchestra.run.smithi083.stdout: 5 11640 14.20 MB/sec warmup 28 sec latency 57.425 ms 2024-09-18T07:26:19.682 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:19 smithi083 ceph-mon[23711]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:19.682 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:19 smithi083 ceph-mon[23711]: pgmap v255: 97 pgs: 97 active+clean; 145 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 609 op/s 2024-09-18T07:26:19.682 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:19 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/880791957' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:26:20.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:19 smithi096 ceph-mon[25928]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:19 smithi096 ceph-mon[25928]: pgmap v255: 97 pgs: 97 active+clean; 145 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 609 op/s 2024-09-18T07:26:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:19 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/880791957' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:26:20.472 INFO:teuthology.orchestra.run.smithi083.stdout: 5 11918 13.97 MB/sec warmup 29 sec latency 53.827 ms 2024-09-18T07:26:21.472 INFO:teuthology.orchestra.run.smithi083.stdout: 5 12372 13.93 MB/sec warmup 30 sec latency 42.769 ms 2024-09-18T07:26:21.709 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:21 smithi083 ceph-mon[23711]: pgmap v256: 97 pgs: 97 active+clean; 158 MiB data, 421 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.4 MiB/s wr, 539 op/s 2024-09-18T07:26:21.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:21 smithi096 ceph-mon[25928]: pgmap v256: 97 pgs: 97 active+clean; 158 MiB data, 421 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.4 MiB/s wr, 539 op/s 2024-09-18T07:26:22.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 12793 13.58 MB/sec warmup 31 sec latency 64.065 ms 2024-09-18T07:26:23.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 13063 13.35 MB/sec warmup 32 sec latency 40.931 ms 2024-09-18T07:26:23.800 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:23 smithi083 ceph-mon[23711]: pgmap v257: 97 pgs: 97 active+clean; 156 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 506 op/s 2024-09-18T07:26:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:23 smithi096 ceph-mon[25928]: pgmap v257: 97 pgs: 97 active+clean; 156 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 506 op/s 2024-09-18T07:26:24.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 13492 13.54 MB/sec warmup 33 sec latency 46.141 ms 2024-09-18T07:26:25.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 13983 13.63 MB/sec warmup 34 sec latency 40.116 ms 2024-09-18T07:26:25.848 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:25 smithi083 ceph-mon[23711]: pgmap v258: 97 pgs: 97 active+clean; 152 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 712 op/s 2024-09-18T07:26:26.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:25 smithi096 ceph-mon[25928]: pgmap v258: 97 pgs: 97 active+clean; 152 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 712 op/s 2024-09-18T07:26:26.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 14413 13.84 MB/sec warmup 35 sec latency 41.351 ms 2024-09-18T07:26:27.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 14833 13.55 MB/sec warmup 36 sec latency 42.323 ms 2024-09-18T07:26:27.860 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:27 smithi083 ceph-mon[23711]: pgmap v259: 97 pgs: 97 active+clean; 154 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-09-18T07:26:28.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:27 smithi096 ceph-mon[25928]: pgmap v259: 97 pgs: 97 active+clean; 154 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-09-18T07:26:28.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 15215 13.35 MB/sec warmup 37 sec latency 35.400 ms 2024-09-18T07:26:29.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 15495 13.25 MB/sec warmup 38 sec latency 66.222 ms 2024-09-18T07:26:29.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:29 smithi083 ceph-mon[23711]: pgmap v260: 97 pgs: 97 active+clean; 162 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.1 MiB/s wr, 552 op/s 2024-09-18T07:26:30.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:29 smithi096 ceph-mon[25928]: pgmap v260: 97 pgs: 97 active+clean; 162 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.1 MiB/s wr, 552 op/s 2024-09-18T07:26:30.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 15935 13.17 MB/sec warmup 39 sec latency 40.133 ms 2024-09-18T07:26:31.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 16390 12.93 MB/sec warmup 40 sec latency 65.892 ms 2024-09-18T07:26:31.681 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:31 smithi083 ceph-mon[23711]: pgmap v261: 97 pgs: 97 active+clean; 171 MiB data, 571 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.3 MiB/s wr, 617 op/s 2024-09-18T07:26:31.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:31 smithi096 ceph-mon[25928]: pgmap v261: 97 pgs: 97 active+clean; 171 MiB data, 571 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.3 MiB/s wr, 617 op/s 2024-09-18T07:26:32.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 16753 12.88 MB/sec warmup 41 sec latency 40.714 ms 2024-09-18T07:26:33.478 INFO:teuthology.orchestra.run.smithi083.stdout: 5 17390 13.23 MB/sec warmup 42 sec latency 33.325 ms 2024-09-18T07:26:33.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:33 smithi083 ceph-mon[23711]: pgmap v262: 97 pgs: 97 active+clean; 169 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-18T07:26:34.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:33 smithi096 ceph-mon[25928]: pgmap v262: 97 pgs: 97 active+clean; 169 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-18T07:26:34.484 INFO:teuthology.orchestra.run.smithi083.stdout: 5 17925 13.46 MB/sec warmup 43 sec latency 43.890 ms 2024-09-18T07:26:35.484 INFO:teuthology.orchestra.run.smithi083.stdout: 5 18154 13.23 MB/sec warmup 44 sec latency 58.878 ms 2024-09-18T07:26:35.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:35 smithi083 ceph-mon[23711]: pgmap v263: 97 pgs: 97 active+clean; 170 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 643 op/s 2024-09-18T07:26:35.886 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:35 smithi096 ceph-mon[25928]: pgmap v263: 97 pgs: 97 active+clean; 170 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 643 op/s 2024-09-18T07:26:36.484 INFO:teuthology.orchestra.run.smithi083.stdout: 5 18444 12.98 MB/sec warmup 45 sec latency 98.123 ms 2024-09-18T07:26:37.484 INFO:teuthology.orchestra.run.smithi083.stdout: 5 18968 12.95 MB/sec warmup 46 sec latency 33.150 ms 2024-09-18T07:26:37.848 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:37 smithi083 ceph-mon[23711]: pgmap v264: 97 pgs: 97 active+clean; 175 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.8 MiB/s wr, 531 op/s 2024-09-18T07:26:38.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:37 smithi096 ceph-mon[25928]: pgmap v264: 97 pgs: 97 active+clean; 175 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.8 MiB/s wr, 531 op/s 2024-09-18T07:26:38.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 19718 13.01 MB/sec warmup 47 sec latency 23.996 ms 2024-09-18T07:26:39.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 20325 13.01 MB/sec warmup 48 sec latency 41.884 ms 2024-09-18T07:26:39.865 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:39 smithi083 ceph-mon[23711]: pgmap v265: 97 pgs: 97 active+clean; 179 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 563 op/s 2024-09-18T07:26:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:39 smithi096 ceph-mon[25928]: pgmap v265: 97 pgs: 97 active+clean; 179 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 563 op/s 2024-09-18T07:26:40.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 20846 13.20 MB/sec warmup 49 sec latency 55.649 ms 2024-09-18T07:26:41.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 21287 13.31 MB/sec warmup 50 sec latency 53.294 ms 2024-09-18T07:26:41.588 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:41 smithi083 ceph-mon[23711]: pgmap v266: 97 pgs: 97 active+clean; 182 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.8 MiB/s wr, 707 op/s 2024-09-18T07:26:41.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:41 smithi096 ceph-mon[25928]: pgmap v266: 97 pgs: 97 active+clean; 182 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.8 MiB/s wr, 707 op/s 2024-09-18T07:26:42.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 21567 13.24 MB/sec warmup 51 sec latency 53.989 ms 2024-09-18T07:26:43.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 21749 13.04 MB/sec warmup 52 sec latency 58.530 ms 2024-09-18T07:26:43.874 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:43 smithi083 ceph-mon[23711]: pgmap v267: 97 pgs: 97 active+clean; 182 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 556 op/s 2024-09-18T07:26:44.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:43 smithi096 ceph-mon[25928]: pgmap v267: 97 pgs: 97 active+clean; 182 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 556 op/s 2024-09-18T07:26:44.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 21956 12.82 MB/sec warmup 53 sec latency 89.369 ms 2024-09-18T07:26:44.901 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:26:44.901 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:26:45.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:26:45.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:26:45.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 22248 12.65 MB/sec warmup 54 sec latency 43.353 ms 2024-09-18T07:26:45.706 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:45 smithi083 ceph-mon[23711]: pgmap v268: 97 pgs: 97 active+clean; 192 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 747 op/s 2024-09-18T07:26:46.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:45 smithi096 ceph-mon[25928]: pgmap v268: 97 pgs: 97 active+clean; 192 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 747 op/s 2024-09-18T07:26:46.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 22506 12.55 MB/sec warmup 55 sec latency 55.273 ms 2024-09-18T07:26:47.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 22913 12.53 MB/sec warmup 56 sec latency 53.264 ms 2024-09-18T07:26:47.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:47 smithi083 ceph-mon[23711]: pgmap v269: 97 pgs: 97 active+clean; 191 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 566 op/s 2024-09-18T07:26:48.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:47 smithi096 ceph-mon[25928]: pgmap v269: 97 pgs: 97 active+clean; 191 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 566 op/s 2024-09-18T07:26:48.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 23326 12.41 MB/sec warmup 57 sec latency 86.148 ms 2024-09-18T07:26:49.400 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:26:49.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 23692 12.29 MB/sec warmup 58 sec latency 51.657 ms 2024-09-18T07:26:49.904 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:49 smithi083 ceph-mon[23711]: pgmap v270: 97 pgs: 97 active+clean; 193 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 615 op/s 2024-09-18T07:26:50.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:49 smithi096 ceph-mon[25928]: pgmap v270: 97 pgs: 97 active+clean; 193 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 615 op/s 2024-09-18T07:26:50.247 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:26:50.247 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (6m) 61s ago 7m 24.6M - ba2b418f427c 376c77ec4596 2024-09-18T07:26:50.247 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (7m) 61s ago 7m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:26:50.247 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (6m) 63s ago 6m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:26:50.247 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (5m) 61s ago 7m 49.7M - 8.3.5 dad864ee21e9 1a3cff9a0185 2024-09-18T07:26:50.247 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (92s) 61s ago 91s 15.8M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (89s) 63s ago 88s 13.4M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:9283 running (8m) 61s ago 8m 462M - 17.2.0 e1d6a67b021e 573994092cfb 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283 running (6m) 63s ago 6m 413M - 17.2.0 e1d6a67b021e 6e155b5f7eb8 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (8m) 61s ago 8m 46.9M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (6m) 63s ago 6m 37.4M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (72s) 61s ago 71s 54.3M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (66s) 63s ago 66s 50.8M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (7m) 61s ago 7m 17.3M - 1dbe0e931976 45810a37da0a 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (6m) 63s ago 6m 16.9M - 1dbe0e931976 3d6582ef7aff 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 61s ago 5m 42.7M 3216M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 61s ago 5m 42.6M 3216M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 61s ago 4m 40.3M 3216M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:26:50.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 61s ago 4m 40.7M 3216M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:26:50.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (4m) 63s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:26:50.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (3m) 63s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:26:50.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (3m) 63s ago 3m 43.7M 3984M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:26:50.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (3m) 63s ago 2m 40.9M 3984M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:26:50.249 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (6m) 61s ago 6m 62.3M - 514e6a882f6e 59f8176c5711 2024-09-18T07:26:50.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 23980 12.29 MB/sec warmup 59 sec latency 74.715 ms 2024-09-18T07:26:50.691 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:50 smithi083 ceph-mon[23711]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:50.716 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:26:50.716 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:26:50.716 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:26:50.716 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:26:50.716 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:26:50.717 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:26:51.076 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:26:51.076 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e", 2024-09-18T07:26:51.077 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:26:51.077 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-09-18T07:26:51.077 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "", 2024-09-18T07:26:51.077 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e image" 2024-09-18T07:26:51.077 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:26:51.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:50 smithi096 ceph-mon[25928]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:51.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 24463 12.41 MB/sec warmup 60 sec latency 64.908 ms 2024-09-18T07:26:51.597 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:26:51.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:51 smithi083 ceph-mon[23711]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:51.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:51 smithi083 ceph-mon[23711]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:51.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:51 smithi083 ceph-mon[23711]: pgmap v271: 97 pgs: 97 active+clean; 188 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.9 MiB/s wr, 657 op/s 2024-09-18T07:26:51.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:51 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2237325744' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:26:51.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:51 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3270034386' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:26:52.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:51 smithi096 ceph-mon[25928]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:52.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:51 smithi096 ceph-mon[25928]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:52.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:51 smithi096 ceph-mon[25928]: pgmap v271: 97 pgs: 97 active+clean; 188 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.9 MiB/s wr, 657 op/s 2024-09-18T07:26:52.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:51 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2237325744' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:26:52.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:51 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/3270034386' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:26:52.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 24813 12.45 MB/sec warmup 61 sec latency 118.078 ms 2024-09-18T07:26:52.781 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:52 smithi083 ceph-mon[23711]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:52.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:52 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:52.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:52 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:26:52.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:52 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:53.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:52 smithi096 ceph-mon[25928]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:26:53.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:52 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:53.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:52 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:26:53.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:52 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:26:53.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 24978 12.37 MB/sec warmup 62 sec latency 72.555 ms 2024-09-18T07:26:53.828 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:53 smithi083 ceph-mon[23711]: Upgrade: Target is version 19.3.0-4983-ge7fb7b56 (unknown) 2024-09-18T07:26:53.828 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:53 smithi083 ceph-mon[23711]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9'] 2024-09-18T07:26:53.829 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:53 smithi083 ceph-mon[23711]: Upgrade: Need to upgrade myself (mgr.smithi083.vchvnu) 2024-09-18T07:26:53.829 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:53 smithi083 ceph-mon[23711]: pgmap v272: 97 pgs: 97 active+clean; 189 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 457 op/s 2024-09-18T07:26:53.829 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:53 smithi083 ceph-mon[23711]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 on smithi096 2024-09-18T07:26:54.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:53 smithi096 ceph-mon[25928]: Upgrade: Target is version 19.3.0-4983-ge7fb7b56 (unknown) 2024-09-18T07:26:54.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:53 smithi096 ceph-mon[25928]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9'] 2024-09-18T07:26:54.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:53 smithi096 ceph-mon[25928]: Upgrade: Need to upgrade myself (mgr.smithi083.vchvnu) 2024-09-18T07:26:54.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:53 smithi096 ceph-mon[25928]: pgmap v272: 97 pgs: 97 active+clean; 189 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 457 op/s 2024-09-18T07:26:54.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:53 smithi096 ceph-mon[25928]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 on smithi096 2024-09-18T07:26:54.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 25273 12.28 MB/sec warmup 63 sec latency 61.945 ms 2024-09-18T07:26:55.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 25718 12.15 MB/sec warmup 64 sec latency 36.754 ms 2024-09-18T07:26:55.859 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:55 smithi083 ceph-mon[23711]: pgmap v273: 97 pgs: 97 active+clean; 202 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 572 op/s 2024-09-18T07:26:56.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:55 smithi096 ceph-mon[25928]: pgmap v273: 97 pgs: 97 active+clean; 202 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 572 op/s 2024-09-18T07:26:56.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 26202 12.16 MB/sec warmup 65 sec latency 32.516 ms 2024-09-18T07:26:57.491 INFO:teuthology.orchestra.run.smithi083.stdout: 5 26910 12.17 MB/sec warmup 66 sec latency 40.375 ms 2024-09-18T07:26:57.887 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:57 smithi083 ceph-mon[23711]: pgmap v274: 97 pgs: 97 active+clean; 206 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-09-18T07:26:58.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:57 smithi096 ceph-mon[25928]: pgmap v274: 97 pgs: 97 active+clean; 206 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-09-18T07:26:58.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 27630 12.31 MB/sec warmup 67 sec latency 52.402 ms 2024-09-18T07:26:59.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 28176 12.49 MB/sec warmup 68 sec latency 46.410 ms 2024-09-18T07:26:59.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:26:59 smithi083 ceph-mon[23711]: pgmap v275: 97 pgs: 97 active+clean; 209 MiB data, 874 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 610 op/s 2024-09-18T07:27:00.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:26:59 smithi096 ceph-mon[25928]: pgmap v275: 97 pgs: 97 active+clean; 209 MiB data, 874 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 610 op/s 2024-09-18T07:27:00.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 28566 12.50 MB/sec warmup 69 sec latency 58.342 ms 2024-09-18T07:27:01.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 28937 12.43 MB/sec warmup 70 sec latency 51.894 ms 2024-09-18T07:27:01.707 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:01 smithi083 ceph-mon[23711]: pgmap v276: 97 pgs: 97 active+clean; 208 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.8 MiB/s wr, 523 op/s 2024-09-18T07:27:01.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:01 smithi096 ceph-mon[25928]: pgmap v276: 97 pgs: 97 active+clean; 208 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.8 MiB/s wr, 523 op/s 2024-09-18T07:27:02.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 29428 12.38 MB/sec warmup 71 sec latency 36.511 ms 2024-09-18T07:27:03.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 29960 12.37 MB/sec warmup 72 sec latency 79.312 ms 2024-09-18T07:27:03.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:03 smithi083 ceph-mon[23711]: pgmap v277: 97 pgs: 97 active+clean; 208 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 452 op/s 2024-09-18T07:27:04.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:03 smithi096 ceph-mon[25928]: pgmap v277: 97 pgs: 97 active+clean; 208 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 452 op/s 2024-09-18T07:27:04.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 30545 12.35 MB/sec warmup 73 sec latency 33.062 ms 2024-09-18T07:27:05.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 31119 12.39 MB/sec warmup 74 sec latency 44.541 ms 2024-09-18T07:27:05.767 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:05 smithi083 ceph-mon[23711]: pgmap v278: 97 pgs: 97 active+clean; 217 MiB data, 947 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 753 op/s 2024-09-18T07:27:06.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:05 smithi096 ceph-mon[25928]: pgmap v278: 97 pgs: 97 active+clean; 217 MiB data, 947 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 753 op/s 2024-09-18T07:27:06.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 31639 12.53 MB/sec warmup 75 sec latency 43.841 ms 2024-09-18T07:27:07.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 32038 12.56 MB/sec warmup 76 sec latency 50.494 ms 2024-09-18T07:27:07.807 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:07 smithi083 ceph-mon[23711]: pgmap v279: 97 pgs: 97 active+clean; 217 MiB data, 952 MiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.2 MiB/s wr, 637 op/s 2024-09-18T07:27:08.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:07 smithi096 ceph-mon[25928]: pgmap v279: 97 pgs: 97 active+clean; 217 MiB data, 952 MiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.2 MiB/s wr, 637 op/s 2024-09-18T07:27:08.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 32470 12.54 MB/sec warmup 77 sec latency 61.732 ms 2024-09-18T07:27:09.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 32966 12.48 MB/sec warmup 78 sec latency 43.402 ms 2024-09-18T07:27:09.651 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:09 smithi083 ceph-mon[23711]: pgmap v280: 97 pgs: 97 active+clean; 222 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 683 op/s 2024-09-18T07:27:10.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:09 smithi096 ceph-mon[25928]: pgmap v280: 97 pgs: 97 active+clean; 222 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 683 op/s 2024-09-18T07:27:10.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 33449 12.46 MB/sec warmup 79 sec latency 54.275 ms 2024-09-18T07:27:11.492 INFO:teuthology.orchestra.run.smithi083.stdout: 5 33888 12.40 MB/sec warmup 80 sec latency 43.335 ms 2024-09-18T07:27:11.681 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:11 smithi083 ceph-mon[23711]: pgmap v281: 97 pgs: 97 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.4 MiB/s wr, 641 op/s 2024-09-18T07:27:11.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:11 smithi096 ceph-mon[25928]: pgmap v281: 97 pgs: 97 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.4 MiB/s wr, 641 op/s 2024-09-18T07:27:12.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 34328 12.39 MB/sec warmup 81 sec latency 49.935 ms 2024-09-18T07:27:13.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 34846 12.47 MB/sec warmup 82 sec latency 43.527 ms 2024-09-18T07:27:13.724 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:13 smithi083 ceph-mon[23711]: pgmap v282: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 589 op/s 2024-09-18T07:27:14.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:13 smithi096 ceph-mon[25928]: pgmap v282: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 589 op/s 2024-09-18T07:27:14.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 35668 12.70 MB/sec warmup 83 sec latency 22.313 ms 2024-09-18T07:27:15.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 36096 12.64 MB/sec warmup 84 sec latency 40.299 ms 2024-09-18T07:27:15.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:15 smithi083 ceph-mon[23711]: pgmap v283: 97 pgs: 97 active+clean; 233 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 861 op/s 2024-09-18T07:27:16.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:15 smithi096 ceph-mon[25928]: pgmap v283: 97 pgs: 97 active+clean; 233 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 861 op/s 2024-09-18T07:27:16.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 36526 12.59 MB/sec warmup 85 sec latency 42.317 ms 2024-09-18T07:27:17.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 37247 12.63 MB/sec warmup 86 sec latency 38.286 ms 2024-09-18T07:27:17.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:17 smithi083 ceph-mon[23711]: pgmap v284: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.5 MiB/s wr, 621 op/s 2024-09-18T07:27:18.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:17 smithi096 ceph-mon[25928]: pgmap v284: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.5 MiB/s wr, 621 op/s 2024-09-18T07:27:18.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 37991 12.65 MB/sec warmup 87 sec latency 32.641 ms 2024-09-18T07:27:19.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 38642 12.80 MB/sec warmup 88 sec latency 43.137 ms 2024-09-18T07:27:19.777 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:19 smithi083 ceph-mon[23711]: pgmap v285: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.2 MiB/s wr, 833 op/s 2024-09-18T07:27:20.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:19 smithi096 ceph-mon[25928]: pgmap v285: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.2 MiB/s wr, 833 op/s 2024-09-18T07:27:20.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 39184 12.90 MB/sec warmup 89 sec latency 39.669 ms 2024-09-18T07:27:21.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 39543 12.85 MB/sec warmup 90 sec latency 40.215 ms 2024-09-18T07:27:21.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:21 smithi083 ceph-mon[23711]: pgmap v286: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 770 op/s 2024-09-18T07:27:21.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:21 smithi096 ceph-mon[25928]: pgmap v286: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 770 op/s 2024-09-18T07:27:22.018 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:27:22.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 39966 12.79 MB/sec warmup 91 sec latency 45.001 ms 2024-09-18T07:27:22.953 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:27:22.953 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (6m) 93s ago 8m 24.6M - ba2b418f427c 376c77ec4596 2024-09-18T07:27:22.954 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (8m) 93s ago 7m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:27:22.954 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (7m) 96s ago 7m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:27:22.954 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (6m) 93s ago 7m 49.7M - 8.3.5 dad864ee21e9 1a3cff9a0185 2024-09-18T07:27:22.954 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (2m) 93s ago 2m 15.8M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:27:22.954 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (2m) 96s ago 2m 13.4M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:27:22.954 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:9283 running (9m) 93s ago 9m 462M - 17.2.0 e1d6a67b021e 573994092cfb 2024-09-18T07:27:22.954 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283 running (7m) 96s ago 7m 413M - 17.2.0 e1d6a67b021e 6e155b5f7eb8 2024-09-18T07:27:22.955 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (9m) 93s ago 9m 46.9M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:27:22.955 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (7m) 96s ago 7m 37.4M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:27:22.955 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (104s) 93s ago 104s 54.3M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:27:22.955 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (99s) 96s ago 98s 50.8M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:27:22.955 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (7m) 93s ago 7m 17.3M - 1dbe0e931976 45810a37da0a 2024-09-18T07:27:22.955 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (7m) 96s ago 7m 16.9M - 1dbe0e931976 3d6582ef7aff 2024-09-18T07:27:22.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 93s ago 6m 42.7M 3216M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:27:22.955 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 93s ago 5m 42.6M 3216M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:27:22.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 93s ago 5m 40.3M 3216M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:27:22.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 93s ago 5m 40.7M 3216M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:27:22.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (4m) 96s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:27:22.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (4m) 96s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:27:22.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (3m) 96s ago 3m 43.7M 3984M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:27:22.956 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (3m) 96s ago 3m 40.9M 3984M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:27:22.956 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (6m) 93s ago 6m 62.3M - 514e6a882f6e 59f8176c5711 2024-09-18T07:27:23.430 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:27:23.430 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:27:23.430 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:27:23.430 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:27:23.431 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:27:23.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 40345 12.72 MB/sec warmup 92 sec latency 37.216 ms 2024-09-18T07:27:23.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:23 smithi083 ceph-mon[23711]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:23.837 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:23 smithi083 ceph-mon[23711]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:23.837 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:23 smithi083 ceph-mon[23711]: pgmap v287: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.9 MiB/s wr, 753 op/s 2024-09-18T07:27:23.837 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:23 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/647113466' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:27:23.918 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:27:23.918 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:27:23.918 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:27:23.919 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-09-18T07:27:23.919 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "0/23 daemons upgraded", 2024-09-18T07:27:23.919 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 image on host smithi096" 2024-09-18T07:27:23.919 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:27:24.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:23 smithi096 ceph-mon[25928]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:24.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:23 smithi096 ceph-mon[25928]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:23 smithi096 ceph-mon[25928]: pgmap v287: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.9 MiB/s wr, 753 op/s 2024-09-18T07:27:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:23 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/647113466' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:27:24.413 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:27:24.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 40701 12.67 MB/sec warmup 93 sec latency 52.608 ms 2024-09-18T07:27:24.886 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:24 smithi083 ceph-mon[23711]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:24.886 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:24 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/918630175' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:27:25.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:24 smithi096 ceph-mon[25928]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:25.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:24 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/918630175' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:27:25.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 41092 12.63 MB/sec warmup 94 sec latency 75.094 ms 2024-09-18T07:27:25.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:25 smithi083 ceph-mon[23711]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:25.676 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:25 smithi083 ceph-mon[23711]: pgmap v288: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 933 op/s 2024-09-18T07:27:26.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:25 smithi096 ceph-mon[25928]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:26.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:25 smithi096 ceph-mon[25928]: pgmap v288: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 933 op/s 2024-09-18T07:27:26.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 41536 12.60 MB/sec warmup 95 sec latency 50.897 ms 2024-09-18T07:27:27.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 42076 12.69 MB/sec warmup 96 sec latency 50.856 ms 2024-09-18T07:27:27.711 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:27 smithi083 ceph-mon[23711]: pgmap v289: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 659 op/s 2024-09-18T07:27:28.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:27 smithi096 ceph-mon[25928]: pgmap v289: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 659 op/s 2024-09-18T07:27:28.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 42559 12.76 MB/sec warmup 97 sec latency 35.759 ms 2024-09-18T07:27:29.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 42910 12.72 MB/sec warmup 98 sec latency 37.510 ms 2024-09-18T07:27:29.765 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:29 smithi083 ceph-mon[23711]: pgmap v290: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.6 MiB/s wr, 727 op/s 2024-09-18T07:27:30.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:29 smithi096 ceph-mon[25928]: pgmap v290: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.6 MiB/s wr, 727 op/s 2024-09-18T07:27:30.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 43260 12.67 MB/sec warmup 99 sec latency 43.090 ms 2024-09-18T07:27:31.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 43644 12.62 MB/sec warmup 100 sec latency 75.168 ms 2024-09-18T07:27:31.721 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:31 smithi083 ceph-mon[23711]: pgmap v291: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.9 MiB/s wr, 629 op/s 2024-09-18T07:27:31.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:31 smithi096 ceph-mon[25928]: pgmap v291: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.9 MiB/s wr, 629 op/s 2024-09-18T07:27:32.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 44218 12.60 MB/sec warmup 101 sec latency 36.008 ms 2024-09-18T07:27:33.493 INFO:teuthology.orchestra.run.smithi083.stdout: 5 44786 12.63 MB/sec warmup 102 sec latency 95.265 ms 2024-09-18T07:27:33.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:33 smithi083 ceph-mon[23711]: pgmap v292: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.3 MiB/s wr, 595 op/s 2024-09-18T07:27:34.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:33 smithi096 ceph-mon[25928]: pgmap v292: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.3 MiB/s wr, 595 op/s 2024-09-18T07:27:34.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 45242 12.60 MB/sec warmup 103 sec latency 53.841 ms 2024-09-18T07:27:35.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 45604 12.62 MB/sec warmup 104 sec latency 44.029 ms 2024-09-18T07:27:35.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:35 smithi083 ceph-mon[23711]: pgmap v293: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 642 op/s 2024-09-18T07:27:36.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:35 smithi096 ceph-mon[25928]: pgmap v293: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 642 op/s 2024-09-18T07:27:36.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 46100 12.69 MB/sec warmup 105 sec latency 47.366 ms 2024-09-18T07:27:37.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 46503 12.67 MB/sec warmup 106 sec latency 40.578 ms 2024-09-18T07:27:37.822 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:37 smithi083 ceph-mon[23711]: pgmap v294: 97 pgs: 97 active+clean; 267 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 539 op/s 2024-09-18T07:27:38.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:37 smithi096 ceph-mon[25928]: pgmap v294: 97 pgs: 97 active+clean; 267 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 539 op/s 2024-09-18T07:27:38.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 46938 12.65 MB/sec warmup 107 sec latency 47.754 ms 2024-09-18T07:27:39.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 47384 12.60 MB/sec warmup 108 sec latency 48.791 ms 2024-09-18T07:27:39.855 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:39 smithi083 ceph-mon[23711]: pgmap v295: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.7 MiB/s wr, 672 op/s 2024-09-18T07:27:40.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:39 smithi096 ceph-mon[25928]: pgmap v295: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.7 MiB/s wr, 672 op/s 2024-09-18T07:27:40.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 47902 12.59 MB/sec warmup 109 sec latency 32.803 ms 2024-09-18T07:27:41.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 48511 12.60 MB/sec warmup 110 sec latency 37.428 ms 2024-09-18T07:27:41.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:41 smithi083 ceph-mon[23711]: pgmap v296: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.8 MiB/s wr, 644 op/s 2024-09-18T07:27:41.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:41 smithi096 ceph-mon[25928]: pgmap v296: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.8 MiB/s wr, 644 op/s 2024-09-18T07:27:42.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 49041 12.66 MB/sec warmup 111 sec latency 34.013 ms 2024-09-18T07:27:43.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 49642 12.75 MB/sec warmup 112 sec latency 51.894 ms 2024-09-18T07:27:43.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:43 smithi083 ceph-mon[23711]: pgmap v297: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 529 op/s 2024-09-18T07:27:43.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:43 smithi096 ceph-mon[25928]: pgmap v297: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 529 op/s 2024-09-18T07:27:44.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 50036 12.73 MB/sec warmup 113 sec latency 71.085 ms 2024-09-18T07:27:44.693 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:27:44.693 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:44 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:27:44.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:27:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:44 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:27:45.501 INFO:teuthology.orchestra.run.smithi083.stdout: 5 50533 12.71 MB/sec warmup 114 sec latency 61.532 ms 2024-09-18T07:27:45.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:45 smithi083 ceph-mon[23711]: pgmap v298: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 777 op/s 2024-09-18T07:27:46.096 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:45 smithi096 ceph-mon[25928]: pgmap v298: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 777 op/s 2024-09-18T07:27:46.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 50998 12.66 MB/sec warmup 115 sec latency 71.477 ms 2024-09-18T07:27:47.229 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:47 smithi083 ceph-mon[23711]: Upgrade: Updating mgr.smithi096.caifnv 2024-09-18T07:27:47.229 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:47 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:27:47.230 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:47 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:27:47.230 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:47 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:27:47.230 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:47 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:27:47.230 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:47 smithi083 ceph-mon[23711]: Deploying daemon mgr.smithi096.caifnv on smithi096 2024-09-18T07:27:47.230 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:47 smithi083 ceph-mon[23711]: pgmap v299: 97 pgs: 97 active+clean; 292 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.0 MiB/s wr, 704 op/s 2024-09-18T07:27:47.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 51618 12.69 MB/sec warmup 116 sec latency 57.613 ms 2024-09-18T07:27:47.504 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:47 smithi096 ceph-mon[25928]: Upgrade: Updating mgr.smithi096.caifnv 2024-09-18T07:27:47.504 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:47 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:27:47.504 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:47 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:27:47.505 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:47 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:27:47.505 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:47 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:27:47.505 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:47 smithi096 ceph-mon[25928]: Deploying daemon mgr.smithi096.caifnv on smithi096 2024-09-18T07:27:47.505 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:47 smithi096 ceph-mon[25928]: pgmap v299: 97 pgs: 97 active+clean; 292 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.0 MiB/s wr, 704 op/s 2024-09-18T07:27:48.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 52209 12.72 MB/sec warmup 117 sec latency 37.332 ms 2024-09-18T07:27:49.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 52813 12.78 MB/sec warmup 118 sec latency 30.186 ms 2024-09-18T07:27:49.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:49 smithi083 ceph-mon[23711]: pgmap v300: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 792 op/s 2024-09-18T07:27:50.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:49 smithi096 ceph-mon[25928]: pgmap v300: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 792 op/s 2024-09-18T07:27:50.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 53313 12.83 MB/sec warmup 119 sec latency 62.715 ms 2024-09-18T07:27:51.574 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:51 smithi083 ceph-mon[23711]: pgmap v301: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.5 MiB/s wr, 706 op/s 2024-09-18T07:27:51.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:51 smithi096 ceph-mon[25928]: pgmap v301: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.5 MiB/s wr, 706 op/s 2024-09-18T07:27:52.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 53995 8.11 MB/sec execute 1 sec latency 54.325 ms 2024-09-18T07:27:53.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 54315 5.88 MB/sec execute 2 sec latency 50.346 ms 2024-09-18T07:27:53.692 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:53 smithi096 ceph-mon[25928]: pgmap v302: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.4 MiB/s wr, 607 op/s 2024-09-18T07:27:53.858 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:53 smithi083 ceph-mon[23711]: pgmap v302: 97 pgs: 97 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.4 MiB/s wr, 607 op/s 2024-09-18T07:27:54.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 54627 6.58 MB/sec execute 3 sec latency 98.261 ms 2024-09-18T07:27:54.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:54 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:27:54.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:54 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:27:54.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:54 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:27:54.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:54 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:27:54.923 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:27:55.070 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:54 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:27:55.070 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:54 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:27:55.070 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:54 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:27:55.070 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:54 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:27:55.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 55248 8.59 MB/sec execute 4 sec latency 31.248 ms 2024-09-18T07:27:55.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:55 smithi083 ceph-mon[23711]: pgmap v303: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 859 op/s 2024-09-18T07:27:55.882 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:27:55.882 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (7m) 2m ago 8m 24.6M - ba2b418f427c 376c77ec4596 2024-09-18T07:27:55.882 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (8m) 2m ago 8m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:27:55.882 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (8m) 2m ago 8m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:27:55.882 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (6m) 2m ago 8m 49.7M - 8.3.5 dad864ee21e9 1a3cff9a0185 2024-09-18T07:27:55.882 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (2m) 2m ago 2m 15.8M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (2m) 2m ago 2m 13.4M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:9283 running (9m) 2m ago 9m 462M - 17.2.0 e1d6a67b021e 573994092cfb 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283 starting - - - - 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (9m) 2m ago 10m 46.9M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (7m) 2m ago 7m 37.4M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (2m) 2m ago 2m 54.3M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (2m) 2m ago 2m 50.8M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (8m) 2m ago 8m 17.3M - 1dbe0e931976 45810a37da0a 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (7m) 2m ago 7m 16.9M - 1dbe0e931976 3d6582ef7aff 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 2m ago 7m 42.7M 3216M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 2m ago 6m 42.6M 3216M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 2m ago 6m 40.3M 3216M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:27:55.883 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 2m ago 5m 40.7M 3216M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:27:55.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (5m) 2m ago 5m 39.4M 3984M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:27:55.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (4m) 2m ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:27:55.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (4m) 2m ago 4m 43.7M 3984M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:27:55.884 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (4m) 2m ago 4m 40.9M 3984M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:27:55.884 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (7m) 2m ago 7m 62.3M - 514e6a882f6e 59f8176c5711 2024-09-18T07:27:56.016 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:55 smithi096 ceph-mon[25928]: pgmap v303: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 859 op/s 2024-09-18T07:27:56.486 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:27:56.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:27:56.486 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:27:56.486 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:27:56.486 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:27:56.487 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:27:56.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 55702 8.84 MB/sec execute 5 sec latency 67.611 ms 2024-09-18T07:27:56.797 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:56 smithi083 ceph-mon[23711]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:56.797 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:56 smithi083 ceph-mon[23711]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:56.797 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:56 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/3842457142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:27:56.981 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:56 smithi096 ceph-mon[25928]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:56.981 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:56 smithi096 ceph-mon[25928]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:56.981 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:56 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/3842457142' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:27:57.008 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:27:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:27:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:27:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-09-18T07:27:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "", 2024-09-18T07:27:57.009 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mgr daemons" 2024-09-18T07:27:57.009 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:27:57.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 56241 10.96 MB/sec execute 6 sec latency 69.234 ms 2024-09-18T07:27:57.531 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:27:57.807 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:57 smithi083 ceph-mon[23711]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:57.807 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:57 smithi083 ceph-mon[23711]: pgmap v304: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 662 op/s 2024-09-18T07:27:57.807 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:57 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:27:57.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:57 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2954214216' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:27:57.967 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:57 smithi096 ceph-mon[25928]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:57.968 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:57 smithi096 ceph-mon[25928]: pgmap v304: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 662 op/s 2024-09-18T07:27:57.968 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:57 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:27:57.968 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:57 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2954214216' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:27:58.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 56694 11.42 MB/sec execute 7 sec latency 65.029 ms 2024-09-18T07:27:58.821 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:58 smithi083 ceph-mon[23711]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:58.821 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:58 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:27:59.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:58 smithi096 ceph-mon[25928]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:27:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:58 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:27:59.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57037 11.56 MB/sec execute 8 sec latency 57.334 ms 2024-09-18T07:27:59.842 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:27:59 smithi083 ceph-mon[23711]: pgmap v305: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.0 MiB/s wr, 675 op/s 2024-09-18T07:28:00.024 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:27:59 smithi096 ceph-mon[25928]: pgmap v305: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.0 MiB/s wr, 675 op/s 2024-09-18T07:28:00.501 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57311 11.04 MB/sec execute 9 sec latency 38.925 ms 2024-09-18T07:28:00.888 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:00 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/crt"}]: dispatch 2024-09-18T07:28:00.889 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:00 smithi083 ceph-mon[23711]: Standby manager daemon smithi096.caifnv restarted 2024-09-18T07:28:00.889 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:00 smithi083 ceph-mon[23711]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:28:00.889 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:00 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:28:00.889 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:00 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/key"}]: dispatch 2024-09-18T07:28:00.889 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:00 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:28:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:00 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/crt"}]: dispatch 2024-09-18T07:28:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:00 smithi096 ceph-mon[25928]: Standby manager daemon smithi096.caifnv restarted 2024-09-18T07:28:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:00 smithi096 ceph-mon[25928]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:28:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:00 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:28:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:00 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/key"}]: dispatch 2024-09-18T07:28:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:00 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:28:01.500 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57672 10.53 MB/sec execute 10 sec latency 52.381 ms 2024-09-18T07:28:01.896 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:01 smithi083 ceph-mon[23711]: pgmap v306: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.7 MiB/s wr, 593 op/s 2024-09-18T07:28:01.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:01 smithi083 ceph-mon[23711]: mgrmap e22: smithi083.vchvnu(active, since 8m), standbys: smithi096.caifnv 2024-09-18T07:28:01.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:28:01.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:28:01.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:28:01.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:28:01.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:01 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr fail", "who": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:28:01.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:01 smithi083 ceph-mon[23711]: osdmap e71: 8 total, 8 up, 8 in 2024-09-18T07:28:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:01 smithi096 ceph-mon[25928]: pgmap v306: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.7 MiB/s wr, 593 op/s 2024-09-18T07:28:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:01 smithi096 ceph-mon[25928]: mgrmap e22: smithi083.vchvnu(active, since 8m), standbys: smithi096.caifnv 2024-09-18T07:28:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:28:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:28:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' 2024-09-18T07:28:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:28:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:01 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr fail", "who": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:28:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:01 smithi096 ceph-mon[25928]: osdmap e71: 8 total, 8 up, 8 in 2024-09-18T07:28:02.501 INFO:teuthology.orchestra.run.smithi083.stdout: 5 57984 10.08 MB/sec execute 11 sec latency 50.951 ms 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: Upgrade: Need to upgrade myself (mgr.smithi083.vchvnu) 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: Failing over to other MGR 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: Standby manager daemon smithi083.vchvnu started 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "mgr fail", "who": "smithi083.vchvnu"}]': finished 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: mgrmap e23: smithi096.caifnv(active, starting, since 1.0074s), standbys: smithi083.vchvnu 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"}]: dispatch 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"}]: dispatch 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr metadata", "who": "smithi096.caifnv", "id": "smithi096.caifnv"}]: dispatch 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:28:02.906 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-18T07:28:02.907 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:02 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: Upgrade: Need to upgrade myself (mgr.smithi083.vchvnu) 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: Failing over to other MGR 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: Standby manager daemon smithi083.vchvnu started 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.14200 172.21.15.83:0/3007360671' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "mgr fail", "who": "smithi083.vchvnu"}]': finished 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: mgrmap e23: smithi096.caifnv(active, starting, since 1.0074s), standbys: smithi083.vchvnu 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"}]: dispatch 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"}]: dispatch 2024-09-18T07:28:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr metadata", "who": "smithi096.caifnv", "id": "smithi096.caifnv"}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-18T07:28:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:02 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-18T07:28:03.501 INFO:teuthology.orchestra.run.smithi083.stdout: 5 58414 10.23 MB/sec execute 12 sec latency 31.588 ms 2024-09-18T07:28:03.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:03 smithi083 ceph-mon[23711]: Manager daemon smithi096.caifnv is now available 2024-09-18T07:28:04.105 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:03 smithi096 ceph-mon[25928]: Manager daemon smithi096.caifnv is now available 2024-09-18T07:28:04.501 INFO:teuthology.orchestra.run.smithi083.stdout: 5 58942 10.10 MB/sec execute 13 sec latency 36.108 ms 2024-09-18T07:28:04.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: mgrmap e24: smithi096.caifnv(active, since 2s), standbys: smithi083.vchvnu 2024-09-18T07:28:04.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:04.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:04.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:04.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: Checking for cert/key for grafana.smithi083 2024-09-18T07:28:04.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:04.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:04.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:28:04.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:28:04.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:28:04.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:28:04.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi096.caifnv/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:28:04.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi096.caifnv/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:28:04.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi096.caifnv/trash_purge_schedule"}]: dispatch 2024-09-18T07:28:04.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi096.caifnv/trash_purge_schedule"}]: dispatch 2024-09-18T07:28:04.996 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:04 smithi083 ceph-mon[23711]: Deploying cephadm binary to smithi096 2024-09-18T07:28:05.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: mgrmap e24: smithi096.caifnv(active, since 2s), standbys: smithi083.vchvnu 2024-09-18T07:28:05.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:05.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:05.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:05.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: Checking for cert/key for grafana.smithi083 2024-09-18T07:28:05.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:05.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:05.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:28:05.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:28:05.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:28:05.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:28:05.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi096.caifnv/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:28:05.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi096.caifnv/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:28:05.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi096.caifnv/trash_purge_schedule"}]: dispatch 2024-09-18T07:28:05.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi096.caifnv/trash_purge_schedule"}]: dispatch 2024-09-18T07:28:05.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:04 smithi096 ceph-mon[25928]: Deploying cephadm binary to smithi096 2024-09-18T07:28:05.503 INFO:teuthology.orchestra.run.smithi083.stdout: 5 59290 10.24 MB/sec execute 14 sec latency 83.916 ms 2024-09-18T07:28:06.503 INFO:teuthology.orchestra.run.smithi083.stdout: 5 59844 10.83 MB/sec execute 15 sec latency 33.193 ms 2024-09-18T07:28:06.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:06 smithi083 ceph-mon[23711]: Deploying cephadm binary to smithi083 2024-09-18T07:28:06.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:06 smithi083 ceph-mon[23711]: pgmap v4: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:28:06.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:06 smithi096 ceph-mon[25928]: Deploying cephadm binary to smithi083 2024-09-18T07:28:06.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:06 smithi096 ceph-mon[25928]: pgmap v4: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:28:07.503 INFO:teuthology.orchestra.run.smithi083.stdout: 5 60353 11.23 MB/sec execute 16 sec latency 32.740 ms 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:07 smithi083 ceph-mon[23711]: [18/Sep/2024:07:28:05] ENGINE Bus STARTING 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:07 smithi083 ceph-mon[23711]: [18/Sep/2024:07:28:05] ENGINE Serving on http://172.21.15.96:8765 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:07 smithi083 ceph-mon[23711]: [18/Sep/2024:07:28:06] ENGINE Serving on https://172.21.15.96:7150 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:07 smithi083 ceph-mon[23711]: [18/Sep/2024:07:28:06] ENGINE Bus STARTED 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:07 smithi083 ceph-mon[23711]: [18/Sep/2024:07:28:06] ENGINE Error in HTTPServer.serve 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Traceback (most recent call last): 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self._connections.run(self.expiration_interval) 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self._run(expiration_interval) 2024-09-18T07:28:07.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: s = self.context.wrap_socket( 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: return self.sslsocket_class._create( 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self.do_handshake() 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self._sslobj.do_handshake() 2024-09-18T07:28:07.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-18T07:28:07.785 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:07 smithi083 ceph-mon[23711]: mgrmap e25: smithi096.caifnv(active, since 4s), standbys: smithi083.vchvnu 2024-09-18T07:28:07.846 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:07 smithi096 ceph-mon[25928]: [18/Sep/2024:07:28:05] ENGINE Bus STARTING 2024-09-18T07:28:07.846 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:07 smithi096 ceph-mon[25928]: [18/Sep/2024:07:28:05] ENGINE Serving on http://172.21.15.96:8765 2024-09-18T07:28:07.846 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:07 smithi096 ceph-mon[25928]: [18/Sep/2024:07:28:06] ENGINE Serving on https://172.21.15.96:7150 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:07 smithi096 ceph-mon[25928]: [18/Sep/2024:07:28:06] ENGINE Bus STARTED 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:07 smithi096 ceph-mon[25928]: [18/Sep/2024:07:28:06] ENGINE Error in HTTPServer.serve 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Traceback (most recent call last): 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self._run(expiration_interval) 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-18T07:28:07.847 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-18T07:28:07.848 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: s = self.context.wrap_socket( 2024-09-18T07:28:07.848 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-18T07:28:07.848 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: return self.sslsocket_class._create( 2024-09-18T07:28:07.848 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-18T07:28:07.848 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self.do_handshake() 2024-09-18T07:28:07.848 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-18T07:28:07.848 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self._sslobj.do_handshake() 2024-09-18T07:28:07.848 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-18T07:28:07.848 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:07 smithi096 ceph-mon[25928]: mgrmap e25: smithi096.caifnv(active, since 4s), standbys: smithi083.vchvnu 2024-09-18T07:28:08.503 INFO:teuthology.orchestra.run.smithi083.stdout: 5 60773 11.26 MB/sec execute 17 sec latency 32.493 ms 2024-09-18T07:28:08.642 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:08 smithi096 ceph-mon[25928]: pgmap v5: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:28:08.788 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:08 smithi083 ceph-mon[23711]: pgmap v5: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:28:09.503 INFO:teuthology.orchestra.run.smithi083.stdout: 5 61211 11.15 MB/sec execute 18 sec latency 36.754 ms 2024-09-18T07:28:09.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:09 smithi096 ceph-mon[25928]: mgrmap e26: smithi096.caifnv(active, since 6s), standbys: smithi083.vchvnu 2024-09-18T07:28:09.795 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:09 smithi083 ceph-mon[23711]: mgrmap e26: smithi096.caifnv(active, since 6s), standbys: smithi083.vchvnu 2024-09-18T07:28:10.505 INFO:teuthology.orchestra.run.smithi083.stdout: 5 61557 10.88 MB/sec execute 19 sec latency 31.340 ms 2024-09-18T07:28:10.562 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:10 smithi083 ceph-mon[23711]: pgmap v6: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:28:10.563 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:10 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:10.563 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:10 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:10.563 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:10 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:10.563 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:10 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:10.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:10 smithi096 ceph-mon[25928]: pgmap v6: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:28:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:10 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:10 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:10 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:10 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:11.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 62000 11.07 MB/sec execute 20 sec latency 31.989 ms 2024-09-18T07:28:11.576 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:11 smithi083 ceph-mon[23711]: Standby manager daemon smithi083.vchvnu restarted 2024-09-18T07:28:11.576 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:11 smithi083 ceph-mon[23711]: Standby manager daemon smithi083.vchvnu started 2024-09-18T07:28:11.576 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:11 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.83:0/567381431' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.vchvnu/crt"}]: dispatch 2024-09-18T07:28:11.576 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:11 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.83:0/567381431' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:28:11.577 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:11 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.83:0/567381431' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.vchvnu/key"}]: dispatch 2024-09-18T07:28:11.577 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:11 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.83:0/567381431' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:28:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:11 smithi096 ceph-mon[25928]: Standby manager daemon smithi083.vchvnu restarted 2024-09-18T07:28:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:11 smithi096 ceph-mon[25928]: Standby manager daemon smithi083.vchvnu started 2024-09-18T07:28:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:11 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.83:0/567381431' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.vchvnu/crt"}]: dispatch 2024-09-18T07:28:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:11 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.83:0/567381431' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:28:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:11 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.83:0/567381431' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.vchvnu/key"}]: dispatch 2024-09-18T07:28:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:11 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.83:0/567381431' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:28:12.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 62531 10.88 MB/sec execute 21 sec latency 69.109 ms 2024-09-18T07:28:12.584 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:12 smithi083 ceph-mon[23711]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 5.6 MiB/s wr, 481 op/s 2024-09-18T07:28:12.584 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:12 smithi083 ceph-mon[23711]: mgrmap e27: smithi096.caifnv(active, since 9s), standbys: smithi083.vchvnu 2024-09-18T07:28:12.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:12 smithi096 ceph-mon[25928]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 5.6 MiB/s wr, 481 op/s 2024-09-18T07:28:12.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:12 smithi096 ceph-mon[25928]: mgrmap e27: smithi096.caifnv(active, since 9s), standbys: smithi083.vchvnu 2024-09-18T07:28:13.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 62966 11.13 MB/sec execute 22 sec latency 33.091 ms 2024-09-18T07:28:14.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 63525 11.34 MB/sec execute 23 sec latency 44.037 ms 2024-09-18T07:28:14.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:14 smithi083 ceph-mon[23711]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.3 MiB/s wr, 373 op/s 2024-09-18T07:28:14.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:14 smithi096 ceph-mon[25928]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.3 MiB/s wr, 373 op/s 2024-09-18T07:28:15.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 63923 11.50 MB/sec execute 24 sec latency 80.580 ms 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:15 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:15.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:15.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:15.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:15.642 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:15 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:16.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 64187 11.39 MB/sec execute 25 sec latency 135.502 ms 2024-09-18T07:28:16.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:16 smithi083 ceph-mon[23711]: pgmap v9: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.2 MiB/s wr, 603 op/s 2024-09-18T07:28:16.750 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:16 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi083 to 1936M 2024-09-18T07:28:16.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:16 smithi096 ceph-mon[25928]: pgmap v9: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.2 MiB/s wr, 603 op/s 2024-09-18T07:28:16.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:16 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi083 to 1936M 2024-09-18T07:28:17.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 64447 11.17 MB/sec execute 26 sec latency 219.409 ms 2024-09-18T07:28:18.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 64889 11.02 MB/sec execute 27 sec latency 36.072 ms 2024-09-18T07:28:18.772 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:18 smithi083 ceph-mon[23711]: pgmap v10: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.6 MiB/s wr, 546 op/s 2024-09-18T07:28:18.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:18 smithi096 ceph-mon[25928]: pgmap v10: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.6 MiB/s wr, 546 op/s 2024-09-18T07:28:19.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 65294 11.07 MB/sec execute 28 sec latency 101.588 ms 2024-09-18T07:28:19.786 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:19 smithi083 ceph-mon[23711]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.6 MiB/s wr, 546 op/s 2024-09-18T07:28:19.786 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:19 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:28:19.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:19 smithi096 ceph-mon[25928]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.6 MiB/s wr, 546 op/s 2024-09-18T07:28:19.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:19 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:28:20.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 65954 11.10 MB/sec execute 29 sec latency 32.443 ms 2024-09-18T07:28:21.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 66462 11.29 MB/sec execute 30 sec latency 59.697 ms 2024-09-18T07:28:22.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: pgmap v12: 97 pgs: 97 active+clean; 347 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 8.2 MiB/s wr, 772 op/s 2024-09-18T07:28:22.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.305 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: Adjusting osd_memory_target on smithi096 to 2704M 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: Updating smithi083:/etc/ceph/ceph.conf 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:28:22.306 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:22 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 66999 11.46 MB/sec execute 31 sec latency 47.887 ms 2024-09-18T07:28:22.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: pgmap v12: 97 pgs: 97 active+clean; 347 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 8.2 MiB/s wr, 772 op/s 2024-09-18T07:28:22.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: Adjusting osd_memory_target on smithi096 to 2704M 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: Updating smithi083:/etc/ceph/ceph.conf 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:28:22.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:22.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:22 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:23.314 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:23 smithi083 ceph-mon[23711]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:28:23.314 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:23 smithi083 ceph-mon[23711]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:28:23.315 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:23 smithi083 ceph-mon[23711]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:28:23.315 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:23 smithi083 ceph-mon[23711]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-09-18T07:28:23.315 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:23 smithi083 ceph-mon[23711]: Deploying daemon alertmanager.smithi083 on smithi083 2024-09-18T07:28:23.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 67429 11.54 MB/sec execute 32 sec latency 34.661 ms 2024-09-18T07:28:23.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:23 smithi096 ceph-mon[25928]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:28:23.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:23 smithi096 ceph-mon[25928]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:28:23.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:23 smithi096 ceph-mon[25928]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:28:23.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:23 smithi096 ceph-mon[25928]: Reconfiguring alertmanager.smithi083 (dependencies changed)... 2024-09-18T07:28:23.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:23 smithi096 ceph-mon[25928]: Deploying daemon alertmanager.smithi083 on smithi083 2024-09-18T07:28:24.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:24 smithi083 ceph-mon[23711]: pgmap v13: 97 pgs: 97 active+clean; 347 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-09-18T07:28:24.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 67845 11.58 MB/sec execute 33 sec latency 32.867 ms 2024-09-18T07:28:24.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:24 smithi096 ceph-mon[25928]: pgmap v13: 97 pgs: 97 active+clean; 347 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 496 op/s 2024-09-18T07:28:25.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 68348 11.51 MB/sec execute 34 sec latency 115.337 ms 2024-09-18T07:28:26.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 68804 11.57 MB/sec execute 35 sec latency 32.986 ms 2024-09-18T07:28:26.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:26 smithi083 ceph-mon[23711]: pgmap v14: 97 pgs: 97 active+clean; 355 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 864 op/s 2024-09-18T07:28:26.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:26 smithi096 ceph-mon[25928]: pgmap v14: 97 pgs: 97 active+clean; 355 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 864 op/s 2024-09-18T07:28:27.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 69371 11.54 MB/sec execute 36 sec latency 38.813 ms 2024-09-18T07:28:28.167 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:28:28.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 69976 11.69 MB/sec execute 37 sec latency 35.108 ms 2024-09-18T07:28:28.626 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:28 smithi083 ceph-mon[23711]: pgmap v15: 97 pgs: 97 active+clean; 355 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 593 op/s 2024-09-18T07:28:28.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:28 smithi096 ceph-mon[25928]: pgmap v15: 97 pgs: 97 active+clean; 355 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 593 op/s 2024-09-18T07:28:29.384 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:28:29.384 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (7m) 14s ago 9m 26.2M - 0.23.0 ba2b418f427c 376c77ec4596 2024-09-18T07:28:29.384 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (9m) 14s ago 9m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:28:29.384 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (8m) 19s ago 8m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:28:29.384 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (7m) 14s ago 8m 49.3M - dad864ee21e9 1a3cff9a0185 2024-09-18T07:28:29.384 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (3m) 14s ago 3m 37.1M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:28:29.384 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (3m) 19s ago 3m 14.3M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:28:29.384 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:9283 running (10m) 14s ago 10m 221M - 17.2.0 e1d6a67b021e 573994092cfb 2024-09-18T07:28:29.384 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283 running (36s) 19s ago 8m 494M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 4c0857fabce9 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (10m) 14s ago 10m 52.8M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (8m) 19s ago 8m 41.4M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (2m) 14s ago 2m 239M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (2m) 19s ago 2m 79.9M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (8m) 14s ago 8m 19.6M - 1.3.1 1dbe0e931976 45810a37da0a 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (8m) 19s ago 8m 21.4M - 1.3.1 1dbe0e931976 3d6582ef7aff 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 14s ago 7m 287M 1936M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7m) 14s ago 7m 203M 1936M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 14s ago 6m 178M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6m) 14s ago 6m 223M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (5m) 19s ago 5m 237M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (5m) 19s ago 5m 264M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:28:29.385 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (5m) 19s ago 5m 243M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:28:29.386 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (4m) 19s ago 4m 325M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:28:29.386 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (8m) 14s ago 8m 63.0M - 2.33.4 514e6a882f6e 59f8176c5711 2024-09-18T07:28:29.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 70633 11.97 MB/sec execute 38 sec latency 35.123 ms 2024-09-18T07:28:29.631 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:29 smithi083 ceph-mon[23711]: from='client.14746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:28:29.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:29 smithi096 ceph-mon[25928]: from='client.14746 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:28:30.139 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:28:30.139 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 1 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:28:30.140 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:28:30.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-18T07:28:30.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 1 2024-09-18T07:28:30.141 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:28:30.141 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:28:30.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 71036 11.99 MB/sec execute 39 sec latency 41.494 ms 2024-09-18T07:28:30.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:30 smithi083 ceph-mon[23711]: pgmap v16: 97 pgs: 97 active+clean; 355 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 593 op/s 2024-09-18T07:28:30.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:30 smithi083 ceph-mon[23711]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:28:30.646 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:30 smithi083 ceph-mon[23711]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:28:30.647 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:30 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/367785392' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:28:30.657 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:28:30.657 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:28:30.657 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:28:30.657 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:28:30.657 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-09-18T07:28:30.657 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "1/23 daemons upgraded", 2024-09-18T07:28:30.657 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-09-18T07:28:30.657 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:28:30.657 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:28:30.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:30 smithi096 ceph-mon[25928]: pgmap v16: 97 pgs: 97 active+clean; 355 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 593 op/s 2024-09-18T07:28:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:30 smithi096 ceph-mon[25928]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:28:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:30 smithi096 ceph-mon[25928]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:28:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:30 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/367785392' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:28:31.282 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:28:31.504 INFO:teuthology.orchestra.run.smithi083.stdout: 5 71332 11.89 MB/sec execute 40 sec latency 38.752 ms 2024-09-18T07:28:31.670 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:31 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/789006590' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:28:31.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:31 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/789006590' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:28:32.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 71750 11.78 MB/sec execute 41 sec latency 54.165 ms 2024-09-18T07:28:32.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:32 smithi083 ceph-mon[23711]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:28:32.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:32 smithi083 ceph-mon[23711]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 907 op/s 2024-09-18T07:28:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:32 smithi096 ceph-mon[25928]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:28:32.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:32 smithi096 ceph-mon[25928]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 907 op/s 2024-09-18T07:28:33.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 72095 11.71 MB/sec execute 42 sec latency 65.061 ms 2024-09-18T07:28:34.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 72444 11.66 MB/sec execute 43 sec latency 49.140 ms 2024-09-18T07:28:34.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:34 smithi083 ceph-mon[23711]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 681 op/s 2024-09-18T07:28:34.741 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:34 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:28:34.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:34 smithi096 ceph-mon[25928]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 681 op/s 2024-09-18T07:28:34.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:34 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:28:35.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 72867 11.62 MB/sec execute 44 sec latency 45.008 ms 2024-09-18T07:28:36.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 73365 11.59 MB/sec execute 45 sec latency 32.874 ms 2024-09-18T07:28:36.766 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:36 smithi083 ceph-mon[23711]: pgmap v19: 97 pgs: 97 active+clean; 354 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 927 op/s 2024-09-18T07:28:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:36 smithi096 ceph-mon[25928]: pgmap v19: 97 pgs: 97 active+clean; 354 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 927 op/s 2024-09-18T07:28:37.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 73780 11.69 MB/sec execute 46 sec latency 44.527 ms 2024-09-18T07:28:38.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 74300 11.73 MB/sec execute 47 sec latency 37.270 ms 2024-09-18T07:28:38.592 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:38 smithi083 ceph-mon[23711]: pgmap v20: 97 pgs: 97 active+clean; 354 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 560 op/s 2024-09-18T07:28:38.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:38 smithi096 ceph-mon[25928]: pgmap v20: 97 pgs: 97 active+clean; 354 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 560 op/s 2024-09-18T07:28:39.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 74674 11.78 MB/sec execute 48 sec latency 63.315 ms 2024-09-18T07:28:40.379 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:40 smithi083 ceph-mon[23711]: pgmap v21: 97 pgs: 97 active+clean; 354 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 560 op/s 2024-09-18T07:28:40.379 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:40 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:40.379 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:40 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:40.379 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:40 smithi083 ceph-mon[23711]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-09-18T07:28:40.379 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:40 smithi083 ceph-mon[23711]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-18T07:28:40.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 75049 11.75 MB/sec execute 49 sec latency 43.817 ms 2024-09-18T07:28:40.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:40 smithi096 ceph-mon[25928]: pgmap v21: 97 pgs: 97 active+clean; 354 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.7 MiB/s wr, 560 op/s 2024-09-18T07:28:40.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:40 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:40.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:40 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:40.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:40 smithi096 ceph-mon[25928]: Reconfiguring grafana.smithi083 (dependencies changed)... 2024-09-18T07:28:40.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:40 smithi096 ceph-mon[25928]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-18T07:28:41.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 75556 11.69 MB/sec execute 50 sec latency 39.339 ms 2024-09-18T07:28:42.142 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:42 smithi083 ceph-mon[23711]: pgmap v22: 97 pgs: 97 active+clean; 350 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 815 op/s 2024-09-18T07:28:42.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:42 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:42.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:42 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:42.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:42 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-18T07:28:42.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:42 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-18T07:28:42.143 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:42 smithi083 ceph-mon[23711]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-09-18T07:28:42.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:42 smithi096 ceph-mon[25928]: pgmap v22: 97 pgs: 97 active+clean; 350 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 815 op/s 2024-09-18T07:28:42.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:42 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:42.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:42 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:42.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:42 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-18T07:28:42.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:42 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-18T07:28:42.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:42 smithi096 ceph-mon[25928]: Reconfiguring daemon grafana.smithi083 on smithi083 2024-09-18T07:28:42.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 75937 11.64 MB/sec execute 51 sec latency 64.213 ms 2024-09-18T07:28:43.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 76378 11.67 MB/sec execute 52 sec latency 63.108 ms 2024-09-18T07:28:44.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:44 smithi083 ceph-mon[23711]: pgmap v23: 97 pgs: 97 active+clean; 350 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 500 op/s 2024-09-18T07:28:44.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:44 smithi096 ceph-mon[25928]: pgmap v23: 97 pgs: 97 active+clean; 350 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 500 op/s 2024-09-18T07:28:44.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 76789 11.56 MB/sec execute 53 sec latency 71.655 ms 2024-09-18T07:28:45.510 INFO:teuthology.orchestra.run.smithi083.stdout: 5 77128 11.58 MB/sec execute 54 sec latency 55.448 ms 2024-09-18T07:28:46.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 77547 11.69 MB/sec execute 55 sec latency 51.876 ms 2024-09-18T07:28:46.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:46 smithi083 ceph-mon[23711]: pgmap v24: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 756 op/s 2024-09-18T07:28:46.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:46 smithi096 ceph-mon[25928]: pgmap v24: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 756 op/s 2024-09-18T07:28:47.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 78010 11.67 MB/sec execute 56 sec latency 77.311 ms 2024-09-18T07:28:48.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 78275 11.62 MB/sec execute 57 sec latency 48.170 ms 2024-09-18T07:28:48.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:48 smithi083 ceph-mon[23711]: pgmap v25: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-09-18T07:28:48.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:48 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:48.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:48 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:48.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:48 smithi096 ceph-mon[25928]: pgmap v25: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-09-18T07:28:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:48 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:48 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:49.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 78713 11.59 MB/sec execute 58 sec latency 54.706 ms 2024-09-18T07:28:49.696 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:49 smithi083 ceph-mon[23711]: Reconfiguring node-exporter.smithi083 (dependencies changed)... 2024-09-18T07:28:49.696 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:49 smithi083 ceph-mon[23711]: Deploying daemon node-exporter.smithi083 on smithi083 2024-09-18T07:28:49.696 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:49 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:28:49.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:49 smithi096 ceph-mon[25928]: Reconfiguring node-exporter.smithi083 (dependencies changed)... 2024-09-18T07:28:49.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:49 smithi096 ceph-mon[25928]: Deploying daemon node-exporter.smithi083 on smithi083 2024-09-18T07:28:49.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:49 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:28:50.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 79220 11.57 MB/sec execute 59 sec latency 54.270 ms 2024-09-18T07:28:50.740 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:50 smithi083 ceph-mon[23711]: pgmap v26: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-09-18T07:28:50.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:50 smithi096 ceph-mon[25928]: pgmap v26: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-09-18T07:28:51.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 79690 11.61 MB/sec execute 60 sec latency 40.542 ms 2024-09-18T07:28:52.512 INFO:teuthology.orchestra.run.smithi083.stdout: 5 80306 11.61 MB/sec execute 61 sec latency 45.127 ms 2024-09-18T07:28:52.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:52 smithi083 ceph-mon[23711]: pgmap v27: 97 pgs: 97 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 737 op/s 2024-09-18T07:28:52.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:52 smithi096 ceph-mon[25928]: pgmap v27: 97 pgs: 97 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 737 op/s 2024-09-18T07:28:53.511 INFO:teuthology.orchestra.run.smithi083.stdout: 5 80878 11.77 MB/sec execute 62 sec latency 32.529 ms 2024-09-18T07:28:54.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81382 11.79 MB/sec execute 63 sec latency 38.436 ms 2024-09-18T07:28:54.781 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:54 smithi083 ceph-mon[23711]: pgmap v28: 97 pgs: 97 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-09-18T07:28:54.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:54 smithi096 ceph-mon[25928]: pgmap v28: 97 pgs: 97 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-09-18T07:28:55.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 81756 11.80 MB/sec execute 64 sec latency 35.928 ms 2024-09-18T07:28:56.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 82084 11.80 MB/sec execute 65 sec latency 62.301 ms 2024-09-18T07:28:56.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:56 smithi083 ceph-mon[23711]: pgmap v29: 97 pgs: 97 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 787 op/s 2024-09-18T07:28:56.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:56 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:56.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:56 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:56.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:56 smithi083 ceph-mon[23711]: Reconfiguring prometheus.smithi083 (dependencies changed)... 2024-09-18T07:28:56.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:56 smithi096 ceph-mon[25928]: pgmap v29: 97 pgs: 97 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 787 op/s 2024-09-18T07:28:56.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:56 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:56.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:56 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:28:56.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:56 smithi096 ceph-mon[25928]: Reconfiguring prometheus.smithi083 (dependencies changed)... 2024-09-18T07:28:57.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 82543 11.74 MB/sec execute 66 sec latency 58.218 ms 2024-09-18T07:28:57.601 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:57 smithi083 ceph-mon[23711]: Deploying daemon prometheus.smithi083 on smithi083 2024-09-18T07:28:57.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:57 smithi096 ceph-mon[25928]: Deploying daemon prometheus.smithi083 on smithi083 2024-09-18T07:28:58.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 82886 11.69 MB/sec execute 67 sec latency 40.301 ms 2024-09-18T07:28:58.611 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:28:58 smithi083 ceph-mon[23711]: pgmap v30: 97 pgs: 97 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-09-18T07:28:58.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:28:58 smithi096 ceph-mon[25928]: pgmap v30: 97 pgs: 97 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-09-18T07:28:59.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 83349 11.70 MB/sec execute 68 sec latency 48.172 ms 2024-09-18T07:29:00.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:00 smithi083 ceph-mon[23711]: pgmap v31: 97 pgs: 97 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-09-18T07:29:00.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 83885 11.63 MB/sec execute 69 sec latency 42.312 ms 2024-09-18T07:29:00.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:00 smithi096 ceph-mon[25928]: pgmap v31: 97 pgs: 97 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-09-18T07:29:01.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 84276 11.68 MB/sec execute 70 sec latency 34.695 ms 2024-09-18T07:29:01.806 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:29:02.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 84757 11.77 MB/sec execute 71 sec latency 35.492 ms 2024-09-18T07:29:02.679 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:02 smithi083 ceph-mon[23711]: pgmap v32: 97 pgs: 97 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 782 op/s 2024-09-18T07:29:02.694 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 starting - - - - 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (9m) 48s ago 9m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (9m) 52s ago 9m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 starting - - - - 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (3m) 48s ago 3m 37.1M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (3m) 52s ago 3m 14.3M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:9283 running (10m) 48s ago 10m 221M - 17.2.0 e1d6a67b021e 573994092cfb 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283 running (69s) 52s ago 9m 494M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 4c0857fabce9 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (10m) 48s ago 11m 52.8M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (8m) 52s ago 8m 41.4M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (3m) 48s ago 3m 239M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (3m) 52s ago 3m 79.9M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:29:02.695 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 starting - - - - 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (8m) 52s ago 8m 21.4M - 1.3.1 1dbe0e931976 3d6582ef7aff 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 48s ago 8m 287M 1936M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7m) 48s ago 7m 203M 1936M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7m) 48s ago 7m 178M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (6m) 48s ago 6m 223M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (6m) 52s ago 6m 237M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (6m) 52s ago 6m 264M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (5m) 52s ago 5m 243M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (5m) 52s ago 5m 325M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:29:02.696 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (8m) 48s ago 8m 63.0M - 2.33.4 514e6a882f6e 59f8176c5711 2024-09-18T07:29:02.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:02 smithi096 ceph-mon[25928]: pgmap v32: 97 pgs: 97 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 782 op/s 2024-09-18T07:29:03.236 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:29:03.236 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:29:03.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:29:03.236 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:29:03.236 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:29:03.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-18T07:29:03.236 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 1 2024-09-18T07:29:03.236 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:29:03.236 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 1 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:29:03.237 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:29:03.521 INFO:teuthology.orchestra.run.smithi083.stdout: 5 85179 11.78 MB/sec execute 72 sec latency 41.644 ms 2024-09-18T07:29:03.675 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:29:03.675 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:29:03.675 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:29:03.675 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:29:03.675 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-09-18T07:29:03.675 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "1/23 daemons upgraded", 2024-09-18T07:29:03.675 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-09-18T07:29:03.675 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:29:03.675 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:29:03.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:03 smithi083 ceph-mon[23711]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:03.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:03 smithi083 ceph-mon[23711]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:03.698 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:03 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/878937057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:29:03.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:03 smithi096 ceph-mon[25928]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:03.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:03 smithi096 ceph-mon[25928]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:03.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:03 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/878937057' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:29:04.308 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:29:04.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 85518 11.74 MB/sec execute 73 sec latency 44.911 ms 2024-09-18T07:29:04.714 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:04 smithi083 ceph-mon[23711]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:04.715 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:04 smithi083 ceph-mon[23711]: pgmap v33: 97 pgs: 97 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 555 op/s 2024-09-18T07:29:04.715 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:04 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:29:04.715 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:04 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2197534990' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:29:04.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:04 smithi096 ceph-mon[25928]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:04.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:04 smithi096 ceph-mon[25928]: pgmap v33: 97 pgs: 97 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 555 op/s 2024-09-18T07:29:04.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:04 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:29:04.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:04 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2197534990' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:29:05.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 85944 11.69 MB/sec execute 74 sec latency 70.022 ms 2024-09-18T07:29:05.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:05 smithi096 ceph-mon[25928]: from='client.14786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:05.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:05 smithi083 ceph-mon[23711]: from='client.14786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:06.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 86373 11.70 MB/sec execute 75 sec latency 50.606 ms 2024-09-18T07:29:06.744 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:06 smithi083 ceph-mon[23711]: pgmap v34: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 810 op/s 2024-09-18T07:29:06.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:06 smithi096 ceph-mon[25928]: pgmap v34: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 810 op/s 2024-09-18T07:29:07.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 86796 11.73 MB/sec execute 76 sec latency 77.065 ms 2024-09-18T07:29:08.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 87285 11.65 MB/sec execute 77 sec latency 62.469 ms 2024-09-18T07:29:08.768 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:08 smithi083 ceph-mon[23711]: pgmap v35: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 506 op/s 2024-09-18T07:29:08.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:08 smithi096 ceph-mon[25928]: pgmap v35: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 506 op/s 2024-09-18T07:29:09.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 87723 11.67 MB/sec execute 78 sec latency 42.696 ms 2024-09-18T07:29:10.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 88162 11.72 MB/sec execute 79 sec latency 47.966 ms 2024-09-18T07:29:10.584 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:10 smithi083 ceph-mon[23711]: pgmap v36: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 506 op/s 2024-09-18T07:29:10.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:10 smithi096 ceph-mon[25928]: pgmap v36: 97 pgs: 97 active+clean; 349 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 506 op/s 2024-09-18T07:29:11.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 88514 11.68 MB/sec execute 80 sec latency 61.771 ms 2024-09-18T07:29:12.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 88807 11.67 MB/sec execute 81 sec latency 48.063 ms 2024-09-18T07:29:12.660 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:12 smithi083 ceph-mon[23711]: pgmap v37: 97 pgs: 97 active+clean; 345 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 768 op/s 2024-09-18T07:29:12.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:12 smithi096 ceph-mon[25928]: pgmap v37: 97 pgs: 97 active+clean; 345 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 768 op/s 2024-09-18T07:29:13.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 89202 11.68 MB/sec execute 82 sec latency 62.647 ms 2024-09-18T07:29:14.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 89608 11.62 MB/sec execute 83 sec latency 58.832 ms 2024-09-18T07:29:14.707 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:14 smithi083 ceph-mon[23711]: pgmap v38: 97 pgs: 97 active+clean; 345 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 517 op/s 2024-09-18T07:29:14.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:14 smithi096 ceph-mon[25928]: pgmap v38: 97 pgs: 97 active+clean; 345 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 517 op/s 2024-09-18T07:29:15.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 89925 11.58 MB/sec execute 84 sec latency 39.982 ms 2024-09-18T07:29:16.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 90271 11.53 MB/sec execute 85 sec latency 53.995 ms 2024-09-18T07:29:16.752 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:16 smithi083 ceph-mon[23711]: pgmap v39: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 743 op/s 2024-09-18T07:29:16.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:16 smithi096 ceph-mon[25928]: pgmap v39: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 743 op/s 2024-09-18T07:29:17.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 90569 11.50 MB/sec execute 86 sec latency 59.554 ms 2024-09-18T07:29:18.223 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:18 smithi096 ceph-mon[25928]: pgmap v40: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 488 op/s 2024-09-18T07:29:18.223 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:18 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:18.223 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:18 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:18.223 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:18 smithi096 ceph-mon[25928]: Reconfiguring node-exporter.smithi096 (dependencies changed)... 2024-09-18T07:29:18.223 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:18 smithi096 ceph-mon[25928]: Deploying daemon node-exporter.smithi096 on smithi096 2024-09-18T07:29:18.287 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:18 smithi083 ceph-mon[23711]: pgmap v40: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 488 op/s 2024-09-18T07:29:18.287 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:18 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:18.287 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:18 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:18.287 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:18 smithi083 ceph-mon[23711]: Reconfiguring node-exporter.smithi096 (dependencies changed)... 2024-09-18T07:29:18.288 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:18 smithi083 ceph-mon[23711]: Deploying daemon node-exporter.smithi096 on smithi096 2024-09-18T07:29:18.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 90986 11.46 MB/sec execute 87 sec latency 63.698 ms 2024-09-18T07:29:19.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 91292 11.44 MB/sec execute 88 sec latency 57.983 ms 2024-09-18T07:29:20.106 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:20 smithi083 ceph-mon[23711]: pgmap v41: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 488 op/s 2024-09-18T07:29:20.106 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:20 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:29:20.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:20 smithi096 ceph-mon[25928]: pgmap v41: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 488 op/s 2024-09-18T07:29:20.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:20 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:29:20.523 INFO:teuthology.orchestra.run.smithi083.stdout: 5 91621 11.43 MB/sec execute 89 sec latency 93.775 ms 2024-09-18T07:29:21.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 92040 11.44 MB/sec execute 90 sec latency 66.785 ms 2024-09-18T07:29:22.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 92401 11.46 MB/sec execute 91 sec latency 49.767 ms 2024-09-18T07:29:22.641 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:22 smithi083 ceph-mon[23711]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 703 op/s 2024-09-18T07:29:22.734 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:22 smithi096 ceph-mon[25928]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 703 op/s 2024-09-18T07:29:23.522 INFO:teuthology.orchestra.run.smithi083.stdout: 5 92746 11.45 MB/sec execute 92 sec latency 38.338 ms 2024-09-18T07:29:23.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-18T07:29:23.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:23.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:23.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-18T07:29:23.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-18T07:29:23.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083.front.sepia.ceph.com:9093"}]: dispatch 2024-09-18T07:29:23.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083.front.sepia.ceph.com:9093"}]: dispatch 2024-09-18T07:29:23.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:23.909 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:29:23.910 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:29:23.910 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi083.front.sepia.ceph.com:3000"}]: dispatch 2024-09-18T07:29:23.910 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi083.front.sepia.ceph.com:3000"}]: dispatch 2024-09-18T07:29:23.910 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:23.910 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:29:23.910 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:29:23.910 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi083.front.sepia.ceph.com:9095"}]: dispatch 2024-09-18T07:29:23.910 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi083.front.sepia.ceph.com:9095"}]: dispatch 2024-09-18T07:29:23.911 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:23.911 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:23.911 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:23.911 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:29:23.911 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: Upgrade: Need to upgrade myself (mgr.smithi096.caifnv) 2024-09-18T07:29:23.911 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: Upgrade: Need to upgrade myself (mgr.smithi096.caifnv) 2024-09-18T07:29:23.911 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:23 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-18T07:29:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-18T07:29:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-18T07:29:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083.front.sepia.ceph.com:9093"}]: dispatch 2024-09-18T07:29:24.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi083.front.sepia.ceph.com:9093"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi083.front.sepia.ceph.com:3000"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi083.front.sepia.ceph.com:3000"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi083.front.sepia.ceph.com:9095"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi083.front.sepia.ceph.com:9095"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:24.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:24.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:29:24.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: Upgrade: Need to upgrade myself (mgr.smithi096.caifnv) 2024-09-18T07:29:24.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: Upgrade: Need to upgrade myself (mgr.smithi096.caifnv) 2024-09-18T07:29:24.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:23 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:24.523 INFO:teuthology.orchestra.run.smithi083.stdout: 5 93004 11.36 MB/sec execute 93 sec latency 100.769 ms 2024-09-18T07:29:24.917 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:24 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:24.918 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:24 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:24.918 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:24 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:25.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:24 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:25.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:24 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:25.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:24 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:25.523 INFO:teuthology.orchestra.run.smithi083.stdout: 5 93350 11.32 MB/sec execute 94 sec latency 34.034 ms 2024-09-18T07:29:26.523 INFO:teuthology.orchestra.run.smithi083.stdout: 5 93592 11.27 MB/sec execute 95 sec latency 68.619 ms 2024-09-18T07:29:26.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:26 smithi083 ceph-mon[23711]: pgmap v44: 97 pgs: 97 active+clean; 345 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 657 op/s 2024-09-18T07:29:26.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:26 smithi083 ceph-mon[23711]: Upgrade: Updating mgr.smithi083.vchvnu 2024-09-18T07:29:26.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:26 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:26.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:26 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.vchvnu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:29:26.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:26 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.vchvnu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:29:26.731 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:26 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:29:26.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:26 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:29:26.732 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:26 smithi083 ceph-mon[23711]: Deploying daemon mgr.smithi083.vchvnu on smithi083 2024-09-18T07:29:26.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:26 smithi096 ceph-mon[25928]: pgmap v44: 97 pgs: 97 active+clean; 345 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 657 op/s 2024-09-18T07:29:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:26 smithi096 ceph-mon[25928]: Upgrade: Updating mgr.smithi083.vchvnu 2024-09-18T07:29:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:26 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:26 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.vchvnu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:29:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:26 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi083.vchvnu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:29:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:26 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:29:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:26 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:29:26.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:26 smithi096 ceph-mon[25928]: Deploying daemon mgr.smithi083.vchvnu on smithi083 2024-09-18T07:29:27.523 INFO:teuthology.orchestra.run.smithi083.stdout: 5 93960 11.26 MB/sec execute 96 sec latency 49.645 ms 2024-09-18T07:29:28.523 INFO:teuthology.orchestra.run.smithi083.stdout: 5 94363 11.22 MB/sec execute 97 sec latency 76.240 ms 2024-09-18T07:29:28.808 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:28 smithi083 ceph-mon[23711]: pgmap v45: 97 pgs: 97 active+clean; 345 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-09-18T07:29:28.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:28 smithi096 ceph-mon[25928]: pgmap v45: 97 pgs: 97 active+clean; 345 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-09-18T07:29:29.523 INFO:teuthology.orchestra.run.smithi083.stdout: 5 94785 11.24 MB/sec execute 98 sec latency 65.794 ms 2024-09-18T07:29:30.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 95142 11.24 MB/sec execute 99 sec latency 54.979 ms 2024-09-18T07:29:30.602 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:30 smithi083 ceph-mon[23711]: pgmap v46: 97 pgs: 97 active+clean; 345 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-09-18T07:29:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:30 smithi096 ceph-mon[25928]: pgmap v46: 97 pgs: 97 active+clean; 345 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-09-18T07:29:31.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 95539 11.24 MB/sec execute 100 sec latency 67.298 ms 2024-09-18T07:29:32.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 95791 11.21 MB/sec execute 101 sec latency 63.191 ms 2024-09-18T07:29:32.658 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:32 smithi083 ceph-mon[23711]: pgmap v47: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 631 op/s 2024-09-18T07:29:32.658 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:32 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:32.658 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:32 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:32.658 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:32 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:29:32.658 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:32 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:32.658 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:32 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:32.857 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:32 smithi096 ceph-mon[25928]: pgmap v47: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 631 op/s 2024-09-18T07:29:32.857 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:32 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:32.857 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:32 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:32.857 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:32 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:29:32.857 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:32 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:32.858 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:32 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:33.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 96030 11.17 MB/sec execute 102 sec latency 51.389 ms 2024-09-18T07:29:34.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 96293 11.16 MB/sec execute 103 sec latency 83.945 ms 2024-09-18T07:29:34.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:34 smithi096 ceph-mon[25928]: pgmap v48: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 417 op/s 2024-09-18T07:29:34.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:34 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:29:34.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:34 smithi083 ceph-mon[23711]: pgmap v48: 97 pgs: 97 active+clean; 339 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 417 op/s 2024-09-18T07:29:34.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:34 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:29:34.991 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:29:35.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 96604 11.08 MB/sec execute 104 sec latency 74.822 ms 2024-09-18T07:29:36.180 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:29:36.180 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 starting - - - - 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (10m) 81s ago 10m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (9m) 0s ago 9m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 starting - - - - 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (4m) 81s ago 4m 37.1M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (4m) 0s ago 4m 14.7M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 starting - - - - 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283 running (102s) 0s ago 9m 520M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 4c0857fabce9 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (11m) 81s ago 11m 52.8M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (9m) 0s ago 9m 42.2M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (3m) 81s ago 3m 239M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (3m) 0s ago 3m 85.5M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 starting - - - - 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (14s) 0s ago 9m 7306k - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:29:36.181 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 81s ago 8m 287M 1936M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:29:36.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8m) 81s ago 8m 203M 1936M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:29:36.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7m) 81s ago 7m 178M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:29:36.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 81s ago 7m 223M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:29:36.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (6m) 0s ago 6m 358M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:29:36.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (6m) 0s ago 6m 385M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:29:36.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (6m) 0s ago 6m 323M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:29:36.182 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (5m) 0s ago 5m 464M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:29:36.182 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 starting - - - - 2024-09-18T07:29:36.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 96966 11.08 MB/sec execute 105 sec latency 45.763 ms 2024-09-18T07:29:36.753 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:36 smithi083 ceph-mon[23711]: pgmap v49: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.5 MiB/s wr, 611 op/s 2024-09-18T07:29:36.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:36 smithi083 ceph-mon[23711]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:36.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:36 smithi083 ceph-mon[23711]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:36.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:36 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:36.754 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:36 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:36.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:36 smithi096 ceph-mon[25928]: pgmap v49: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.5 MiB/s wr, 611 op/s 2024-09-18T07:29:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:36 smithi096 ceph-mon[25928]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:36 smithi096 ceph-mon[25928]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:36 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:36 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:36.993 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:29:36.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:29:36.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:29:36.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:29:36.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:29:36.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-18T07:29:36.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 1 2024-09-18T07:29:36.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 1 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:29:36.994 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:29:37.477 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:29:37.477 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:29:37.477 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:29:37.478 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:29:37.478 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-09-18T07:29:37.478 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "", 2024-09-18T07:29:37.478 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mgr daemons", 2024-09-18T07:29:37.478 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:29:37.478 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:29:37.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 97131 11.00 MB/sec execute 106 sec latency 73.276 ms 2024-09-18T07:29:37.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:37 smithi083 ceph-mon[23711]: from='client.14806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:37.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:37 smithi083 ceph-mon[23711]: pgmap v50: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-18T07:29:37.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:37 smithi083 ceph-mon[23711]: mgrmap e28: smithi096.caifnv(active, since 95s), standbys: smithi083.vchvnu 2024-09-18T07:29:37.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:37 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2188968305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:29:37.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:37 smithi083 ceph-mon[23711]: from='client.24525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:37.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:37 smithi096 ceph-mon[25928]: from='client.14806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:37 smithi096 ceph-mon[25928]: pgmap v50: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-18T07:29:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:37 smithi096 ceph-mon[25928]: mgrmap e28: smithi096.caifnv(active, since 95s), standbys: smithi083.vchvnu 2024-09-18T07:29:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:37 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2188968305' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:29:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:37 smithi096 ceph-mon[25928]: from='client.24525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:29:38.103 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:29:38.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 97380 10.96 MB/sec execute 107 sec latency 58.582 ms 2024-09-18T07:29:38.789 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:38 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/2418256078' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:29:39.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:38 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/2418256078' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:29:39.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 97741 10.95 MB/sec execute 108 sec latency 60.629 ms 2024-09-18T07:29:39.814 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:39 smithi083 ceph-mon[23711]: pgmap v51: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-18T07:29:39.814 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:39 smithi083 ceph-mon[23711]: Standby manager daemon smithi083.vchvnu restarted 2024-09-18T07:29:39.814 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:39 smithi083 ceph-mon[23711]: Standby manager daemon smithi083.vchvnu started 2024-09-18T07:29:39.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:39 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.vchvnu/crt"}]: dispatch 2024-09-18T07:29:39.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:39 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:29:39.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:39 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.vchvnu/key"}]: dispatch 2024-09-18T07:29:39.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:39 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:29:40.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:39 smithi096 ceph-mon[25928]: pgmap v51: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-18T07:29:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:39 smithi096 ceph-mon[25928]: Standby manager daemon smithi083.vchvnu restarted 2024-09-18T07:29:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:39 smithi096 ceph-mon[25928]: Standby manager daemon smithi083.vchvnu started 2024-09-18T07:29:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:39 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.vchvnu/crt"}]: dispatch 2024-09-18T07:29:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:39 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:29:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:39 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi083.vchvnu/key"}]: dispatch 2024-09-18T07:29:40.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:39 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:29:40.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 98218 10.95 MB/sec execute 109 sec latency 44.983 ms 2024-09-18T07:29:40.850 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:40 smithi083 ceph-mon[23711]: mgrmap e29: smithi096.caifnv(active, since 98s), standbys: smithi083.vchvnu 2024-09-18T07:29:40.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:40 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:40.851 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:40 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:41.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:40 smithi096 ceph-mon[25928]: mgrmap e29: smithi096.caifnv(active, since 98s), standbys: smithi083.vchvnu 2024-09-18T07:29:41.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:40 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:41.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:40 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:41.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 98527 10.94 MB/sec execute 110 sec latency 52.598 ms 2024-09-18T07:29:42.154 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:42 smithi083 ceph-mon[23711]: pgmap v52: 97 pgs: 97 active+clean; 337 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.7 MiB/s wr, 567 op/s 2024-09-18T07:29:42.154 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:42 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:42.154 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:42 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:42.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:42 smithi096 ceph-mon[25928]: pgmap v52: 97 pgs: 97 active+clean; 337 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.7 MiB/s wr, 567 op/s 2024-09-18T07:29:42.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:42 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:42.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:42 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:42.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 98985 10.96 MB/sec execute 111 sec latency 47.779 ms 2024-09-18T07:29:43.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 99320 10.95 MB/sec execute 112 sec latency 48.715 ms 2024-09-18T07:29:44.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:44 smithi083 ceph-mon[23711]: pgmap v53: 97 pgs: 97 active+clean; 337 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 367 op/s 2024-09-18T07:29:44.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:44 smithi096 ceph-mon[25928]: pgmap v53: 97 pgs: 97 active+clean; 337 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 367 op/s 2024-09-18T07:29:44.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 99766 11.00 MB/sec execute 113 sec latency 48.078 ms 2024-09-18T07:29:45.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 100112 10.95 MB/sec execute 114 sec latency 55.710 ms 2024-09-18T07:29:46.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 100428 10.91 MB/sec execute 115 sec latency 56.550 ms 2024-09-18T07:29:46.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:46 smithi096 ceph-mon[25928]: pgmap v54: 97 pgs: 97 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 598 op/s 2024-09-18T07:29:46.779 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:46 smithi083 ceph-mon[23711]: pgmap v54: 97 pgs: 97 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 598 op/s 2024-09-18T07:29:47.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 100771 10.91 MB/sec execute 116 sec latency 51.595 ms 2024-09-18T07:29:48.084 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: pgmap v55: 97 pgs: 97 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: Upgrade: Need to upgrade myself (mgr.smithi096.caifnv) 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: Upgrade: Need to upgrade myself (mgr.smithi096.caifnv) 2024-09-18T07:29:48.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: Failing over to other MGR 2024-09-18T07:29:48.086 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr fail", "who": "smithi096.caifnv"}]: dispatch 2024-09-18T07:29:48.086 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr fail", "who": "smithi096.caifnv"}]: dispatch 2024-09-18T07:29:48.086 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: osdmap e72: 8 total, 8 up, 8 in 2024-09-18T07:29:48.086 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:48 smithi083 ceph-mon[23711]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:29:48.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: pgmap v55: 97 pgs: 97 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: Upgrade: Need to upgrade myself (mgr.smithi096.caifnv) 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: Upgrade: Need to upgrade myself (mgr.smithi096.caifnv) 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: Failing over to other MGR 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: from='mgr.24425 172.21.15.96:0/95594380' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr fail", "who": "smithi096.caifnv"}]: dispatch 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "mgr fail", "who": "smithi096.caifnv"}]: dispatch 2024-09-18T07:29:48.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: osdmap e72: 8 total, 8 up, 8 in 2024-09-18T07:29:48.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:48 smithi096 ceph-mon[25928]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:29:48.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 101121 10.89 MB/sec execute 117 sec latency 32.271 ms 2024-09-18T07:29:49.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd='[{"prefix": "mgr fail", "who": "smithi096.caifnv"}]': finished 2024-09-18T07:29:49.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: mgrmap e30: smithi083.vchvnu(active, starting, since 0.702848s), standbys: smithi096.caifnv 2024-09-18T07:29:49.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:29:49.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:29:49.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr metadata", "who": "smithi096.caifnv", "id": "smithi096.caifnv"}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-18T07:29:49.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-18T07:29:49.358 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: Manager daemon smithi083.vchvnu is now available 2024-09-18T07:29:49.358 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:29:49.358 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:29:49.358 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:49.358 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:29:49.358 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:29:49.358 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:29:49.358 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:49 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:29:49.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 101420 10.87 MB/sec execute 118 sec latency 74.518 ms 2024-09-18T07:29:49.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.24425 ' entity='mgr.smithi096.caifnv' cmd='[{"prefix": "mgr fail", "who": "smithi096.caifnv"}]': finished 2024-09-18T07:29:49.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: mgrmap e30: smithi083.vchvnu(active, starting, since 0.702848s), standbys: smithi096.caifnv 2024-09-18T07:29:49.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi083"}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata", "id": "smithi096"}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr metadata", "who": "smithi096.caifnv", "id": "smithi096.caifnv"}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-18T07:29:49.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-18T07:29:49.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-18T07:29:49.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-18T07:29:49.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-18T07:29:49.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: Manager daemon smithi083.vchvnu is now available 2024-09-18T07:29:49.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:29:49.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:29:49.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:49.625 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:29:49.625 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"}]: dispatch 2024-09-18T07:29:49.625 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:29:49.625 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:49 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"}]: dispatch 2024-09-18T07:29:50.365 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:50 smithi083 ceph-mon[23711]: mgrmap e31: smithi083.vchvnu(active, since 1.71007s), standbys: smithi096.caifnv 2024-09-18T07:29:50.365 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:50 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:50.365 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:50 smithi083 ceph-mon[23711]: pgmap v3: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:29:50.366 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:50 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:50.366 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:50 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:50.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 101764 10.84 MB/sec execute 119 sec latency 55.665 ms 2024-09-18T07:29:50.561 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:50 smithi096 ceph-mon[25928]: mgrmap e31: smithi083.vchvnu(active, since 1.71007s), standbys: smithi096.caifnv 2024-09-18T07:29:50.561 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:50 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:29:50.561 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:50 smithi096 ceph-mon[25928]: pgmap v3: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:29:50.561 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:50 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:50.561 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:50 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:51.385 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:51 smithi083 ceph-mon[23711]: mgrmap e32: smithi083.vchvnu(active, since 2s), standbys: smithi096.caifnv 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:51 smithi083 ceph-mon[23711]: [18/Sep/2024:07:29:50] ENGINE Bus STARTING 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:51 smithi083 ceph-mon[23711]: [18/Sep/2024:07:29:50] ENGINE Serving on https://172.21.15.83:7150 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:51 smithi083 ceph-mon[23711]: [18/Sep/2024:07:29:50] ENGINE Error in HTTPServer.serve 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Traceback (most recent call last): 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self._connections.run(self.expiration_interval) 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self._run(expiration_interval) 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-18T07:29:51.386 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: s = self.context.wrap_socket( 2024-09-18T07:29:51.387 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-18T07:29:51.387 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: return self.sslsocket_class._create( 2024-09-18T07:29:51.387 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-18T07:29:51.387 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self.do_handshake() 2024-09-18T07:29:51.387 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-18T07:29:51.387 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self._sslobj.do_handshake() 2024-09-18T07:29:51.387 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-18T07:29:51.387 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:51 smithi083 ceph-mon[23711]: [18/Sep/2024:07:29:50] ENGINE Serving on http://172.21.15.83:8765 2024-09-18T07:29:51.387 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:51 smithi083 ceph-mon[23711]: [18/Sep/2024:07:29:50] ENGINE Bus STARTED 2024-09-18T07:29:51.464 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:51 smithi096 ceph-mon[25928]: mgrmap e32: smithi083.vchvnu(active, since 2s), standbys: smithi096.caifnv 2024-09-18T07:29:51.464 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:51 smithi096 ceph-mon[25928]: [18/Sep/2024:07:29:50] ENGINE Bus STARTING 2024-09-18T07:29:51.464 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:51 smithi096 ceph-mon[25928]: [18/Sep/2024:07:29:50] ENGINE Serving on https://172.21.15.83:7150 2024-09-18T07:29:51.464 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:51 smithi096 ceph-mon[25928]: [18/Sep/2024:07:29:50] ENGINE Error in HTTPServer.serve 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Traceback (most recent call last): 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self._run(expiration_interval) 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: s = self.context.wrap_socket( 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-18T07:29:51.465 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: return self.sslsocket_class._create( 2024-09-18T07:29:51.466 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-18T07:29:51.466 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self.do_handshake() 2024-09-18T07:29:51.466 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-18T07:29:51.466 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self._sslobj.do_handshake() 2024-09-18T07:29:51.466 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-18T07:29:51.466 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:51 smithi096 ceph-mon[25928]: [18/Sep/2024:07:29:50] ENGINE Serving on http://172.21.15.83:8765 2024-09-18T07:29:51.466 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:51 smithi096 ceph-mon[25928]: [18/Sep/2024:07:29:50] ENGINE Bus STARTED 2024-09-18T07:29:51.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 102109 10.83 MB/sec execute 120 sec latency 58.385 ms 2024-09-18T07:29:52.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 102658 10.88 MB/sec execute 121 sec latency 74.204 ms 2024-09-18T07:29:53.411 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:53 smithi083 ceph-mon[23711]: pgmap v5: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:29:53.411 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:53 smithi083 ceph-mon[23711]: mgrmap e33: smithi083.vchvnu(active, since 4s), standbys: smithi096.caifnv 2024-09-18T07:29:53.515 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:53 smithi096 ceph-mon[25928]: pgmap v5: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:29:53.516 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:53 smithi096 ceph-mon[25928]: mgrmap e33: smithi083.vchvnu(active, since 4s), standbys: smithi096.caifnv 2024-09-18T07:29:53.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 103112 10.92 MB/sec execute 122 sec latency 50.728 ms 2024-09-18T07:29:54.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 103511 10.90 MB/sec execute 123 sec latency 44.170 ms 2024-09-18T07:29:54.955 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: Standby manager daemon smithi096.caifnv restarted 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/1671496134' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/crt"}]: dispatch 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/1671496134' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/1671496134' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/key"}]: dispatch 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/1671496134' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: pgmap v6: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:54.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:54 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: Standby manager daemon smithi096.caifnv restarted 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/1671496134' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/crt"}]: dispatch 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/1671496134' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/1671496134' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/key"}]: dispatch 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/1671496134' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: pgmap v6: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-18T07:29:55.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:55.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:54 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:55.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 103828 10.86 MB/sec execute 124 sec latency 74.767 ms 2024-09-18T07:29:55.972 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:55 smithi083 ceph-mon[23711]: mgrmap e34: smithi083.vchvnu(active, since 7s), standbys: smithi096.caifnv 2024-09-18T07:29:56.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:55 smithi096 ceph-mon[25928]: mgrmap e34: smithi083.vchvnu(active, since 7s), standbys: smithi096.caifnv 2024-09-18T07:29:56.524 INFO:teuthology.orchestra.run.smithi083.stdout: 5 104253 10.88 MB/sec execute 125 sec latency 51.758 ms 2024-09-18T07:29:56.981 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:56 smithi083 ceph-mon[23711]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 4.5 MiB/s wr, 429 op/s 2024-09-18T07:29:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:56 smithi096 ceph-mon[25928]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 4.5 MiB/s wr, 429 op/s 2024-09-18T07:29:57.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 104672 10.88 MB/sec execute 126 sec latency 48.907 ms 2024-09-18T07:29:58.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 105080 10.86 MB/sec execute 127 sec latency 46.288 ms 2024-09-18T07:29:58.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:58 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:58.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:58 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:58.762 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:58 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:58.763 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:58 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:59.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:58 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:58 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:58 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:58 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:29:59.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 105447 10.85 MB/sec execute 128 sec latency 63.793 ms 2024-09-18T07:29:59.776 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:29:59 smithi083 ceph-mon[23711]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 333 op/s 2024-09-18T07:30:00.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:29:59 smithi096 ceph-mon[25928]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 333 op/s 2024-09-18T07:30:00.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 105779 10.85 MB/sec execute 129 sec latency 43.166 ms 2024-09-18T07:30:00.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:00 smithi083 ceph-mon[23711]: overall HEALTH_OK 2024-09-18T07:30:01.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:00 smithi096 ceph-mon[25928]: overall HEALTH_OK 2024-09-18T07:30:01.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 106142 10.86 MB/sec execute 130 sec latency 47.863 ms 2024-09-18T07:30:01.827 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:01 smithi083 ceph-mon[23711]: pgmap v9: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.3 MiB/s wr, 535 op/s 2024-09-18T07:30:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:01 smithi096 ceph-mon[25928]: pgmap v9: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.3 MiB/s wr, 535 op/s 2024-09-18T07:30:02.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 106532 10.87 MB/sec execute 131 sec latency 31.055 ms 2024-09-18T07:30:02.856 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:02 smithi083 ceph-mon[23711]: pgmap v10: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 487 op/s 2024-09-18T07:30:03.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:02 smithi096 ceph-mon[25928]: pgmap v10: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 487 op/s 2024-09-18T07:30:03.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 106830 10.84 MB/sec execute 132 sec latency 55.444 ms 2024-09-18T07:30:04.138 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:04 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:30:04.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:04 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:30:04.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 107187 10.82 MB/sec execute 133 sec latency 38.779 ms 2024-09-18T07:30:05.155 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:05 smithi083 ceph-mon[23711]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 487 op/s 2024-09-18T07:30:05.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:05 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:05.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:05 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:05.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:05 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:30:05.156 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:05 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:30:05.336 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:05 smithi096 ceph-mon[25928]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 487 op/s 2024-09-18T07:30:05.336 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:05 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:05.336 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:05 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:05.336 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:05 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:30:05.337 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:05 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:30:05.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 107455 10.77 MB/sec execute 134 sec latency 92.237 ms 2024-09-18T07:30:06.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 107853 10.79 MB/sec execute 135 sec latency 39.703 ms 2024-09-18T07:30:06.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: Updating smithi083:/etc/ceph/ceph.conf 2024-09-18T07:30:06.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-18T07:30:06.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:30:06.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:30:06.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:30:06.687 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:30:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:30:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:30:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.688 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:06 smithi083 ceph-mon[23711]: Reconfiguring prometheus.smithi083 (dependencies changed)... 2024-09-18T07:30:06.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: Updating smithi083:/etc/ceph/ceph.conf 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:06.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:06 smithi096 ceph-mon[25928]: Reconfiguring prometheus.smithi083 (dependencies changed)... 2024-09-18T07:30:07.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 108195 10.78 MB/sec execute 136 sec latency 81.604 ms 2024-09-18T07:30:07.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:07 smithi083 ceph-mon[23711]: Reconfiguring daemon prometheus.smithi083 on smithi083 2024-09-18T07:30:07.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:07 smithi083 ceph-mon[23711]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.1 MiB/s wr, 694 op/s 2024-09-18T07:30:07.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:07 smithi096 ceph-mon[25928]: Reconfiguring daemon prometheus.smithi083 on smithi083 2024-09-18T07:30:07.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:07 smithi096 ceph-mon[25928]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.1 MiB/s wr, 694 op/s 2024-09-18T07:30:08.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 108543 10.76 MB/sec execute 137 sec latency 94.285 ms 2024-09-18T07:30:08.676 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:30:09.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 108907 10.73 MB/sec execute 138 sec latency 49.556 ms 2024-09-18T07:30:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:30:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (91s) 12s ago 10m 19.8M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:30:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (10m) 12s ago 10m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:30:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (10m) 15s ago 10m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:30:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (82s) 12s ago 10m 45.2M - dad864ee21e9 06ff04a18f50 2024-09-18T07:30:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (4m) 12s ago 4m 39.5M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:30:09.750 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (4m) 15s ago 4m 15.1M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (38s) 12s ago 11m 492M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283 running (2m) 15s ago 10m 159M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 4c0857fabce9 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (11m) 12s ago 12m 55.4M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (10m) 15s ago 10m 46.3M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (4m) 12s ago 4m 262M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (4m) 15s ago 4m 85.6M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (75s) 12s ago 10m 8250k - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (47s) 15s ago 9m 7545k - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9m) 12s ago 9m 420M 1936M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8m) 12s ago 8m 318M 1936M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8m) 12s ago 8m 252M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 12s ago 7m 388M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:30:09.751 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (7m) 15s ago 7m 375M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:30:09.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (7m) 15s ago 7m 390M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:30:09.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (6m) 15s ago 6m 334M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:30:09.752 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (6m) 15s ago 6m 490M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:30:09.752 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (53s) 12s ago 9m 41.5M - 2.51.0 1d3b7f56885b b2657c99dc0e 2024-09-18T07:30:09.796 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:09 smithi083 ceph-mon[23711]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-09-18T07:30:09.796 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:09 smithi083 ceph-mon[23711]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:10.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:09 smithi096 ceph-mon[25928]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-09-18T07:30:10.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:09 smithi096 ceph-mon[25928]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:10.364 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:30:10.364 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:30:10.364 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:30:10.364 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:30:10.365 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:30:10.366 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:30:10.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 109171 10.70 MB/sec execute 139 sec latency 61.070 ms 2024-09-18T07:30:10.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:10 smithi083 ceph-mon[23711]: from='client.14846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:10.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:10 smithi083 ceph-mon[23711]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:10.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:10 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/1323044904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:30:10.881 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:30:10.881 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:30:10.881 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:30:10.881 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:30:10.881 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:30:10.882 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr" 2024-09-18T07:30:10.882 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:30:10.882 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "2/23 daemons upgraded", 2024-09-18T07:30:10.882 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-09-18T07:30:10.882 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:30:10.882 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:30:11.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:10 smithi096 ceph-mon[25928]: from='client.14846 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:11.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:10 smithi096 ceph-mon[25928]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:11.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:10 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/1323044904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:30:11.497 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:30:11.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 109547 10.74 MB/sec execute 140 sec latency 71.730 ms 2024-09-18T07:30:11.846 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:11 smithi083 ceph-mon[23711]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 655 op/s 2024-09-18T07:30:11.846 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:11 smithi083 ceph-mon[23711]: from='client.? 172.21.15.83:0/805168170' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:30:12.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:11 smithi096 ceph-mon[25928]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 655 op/s 2024-09-18T07:30:12.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:11 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/805168170' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-18T07:30:12.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 109864 10.71 MB/sec execute 141 sec latency 66.373 ms 2024-09-18T07:30:12.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:12 smithi083 ceph-mon[23711]: from='client.14858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:13.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:12 smithi096 ceph-mon[25928]: from='client.14858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:13.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 110150 10.72 MB/sec execute 142 sec latency 59.190 ms 2024-09-18T07:30:13.901 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:13 smithi083 ceph-mon[23711]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-09-18T07:30:14.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:13 smithi096 ceph-mon[25928]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-09-18T07:30:14.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 110534 10.70 MB/sec execute 143 sec latency 35.746 ms 2024-09-18T07:30:15.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 110986 10.69 MB/sec execute 144 sec latency 36.924 ms 2024-09-18T07:30:15.923 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:15 smithi083 ceph-mon[23711]: pgmap v16: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 429 op/s 2024-09-18T07:30:15.923 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:15 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:15.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:15 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:15.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:15 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:30:15.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:15 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:30:15.996 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:15 smithi096 ceph-mon[25928]: pgmap v16: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 429 op/s 2024-09-18T07:30:15.997 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:15 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:15.997 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:15 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:15.997 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:15 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:30:15.997 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:15 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:30:16.525 INFO:teuthology.orchestra.run.smithi083.stdout: 5 111432 10.72 MB/sec execute 145 sec latency 49.388 ms 2024-09-18T07:30:16.776 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:16 smithi083 ceph-mon[23711]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:30:17.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:16 smithi096 ceph-mon[25928]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-18T07:30:17.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 111907 10.74 MB/sec execute 146 sec latency 54.345 ms 2024-09-18T07:30:17.781 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:17 smithi083 ceph-mon[23711]: pgmap v17: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 610 op/s 2024-09-18T07:30:17.781 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:17 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:30:17.781 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:17 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:30:17.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:17 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:30:17.782 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:17 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:30:17.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:17 smithi096 ceph-mon[25928]: pgmap v17: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 610 op/s 2024-09-18T07:30:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:17 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:30:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:17 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-18T07:30:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:17 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-18T07:30:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:17 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:30:18.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 112367 10.72 MB/sec execute 147 sec latency 76.210 ms 2024-09-18T07:30:18.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:18 smithi083 ceph-mon[23711]: Upgrade: Updating mgr.smithi096.caifnv 2024-09-18T07:30:18.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:18 smithi083 ceph-mon[23711]: Deploying daemon mgr.smithi096.caifnv on smithi096 2024-09-18T07:30:19.029 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:18 smithi096 ceph-mon[25928]: Upgrade: Updating mgr.smithi096.caifnv 2024-09-18T07:30:19.029 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:18 smithi096 ceph-mon[25928]: Deploying daemon mgr.smithi096.caifnv on smithi096 2024-09-18T07:30:19.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 112761 10.70 MB/sec execute 148 sec latency 83.755 ms 2024-09-18T07:30:19.833 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:19 smithi083 ceph-mon[23711]: pgmap v18: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-18T07:30:19.833 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:19 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:30:20.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:19 smithi096 ceph-mon[25928]: pgmap v18: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-18T07:30:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:19 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:30:20.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 113211 10.76 MB/sec execute 149 sec latency 54.183 ms 2024-09-18T07:30:21.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 113592 10.74 MB/sec execute 150 sec latency 55.958 ms 2024-09-18T07:30:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:21 smithi096 ceph-mon[25928]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 677 op/s 2024-09-18T07:30:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:21 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:21 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:21.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:21 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:30:21.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:21 smithi083 ceph-mon[23711]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 677 op/s 2024-09-18T07:30:21.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:21 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:21.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:21 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:21.916 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:21 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:30:22.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 113919 10.73 MB/sec execute 151 sec latency 52.365 ms 2024-09-18T07:30:23.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 114177 10.71 MB/sec execute 152 sec latency 50.898 ms 2024-09-18T07:30:23.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:23 smithi096 ceph-mon[25928]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-18T07:30:23.995 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:23 smithi083 ceph-mon[23711]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-18T07:30:24.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 114480 10.70 MB/sec execute 153 sec latency 47.951 ms 2024-09-18T07:30:25.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 114864 10.69 MB/sec execute 154 sec latency 72.867 ms 2024-09-18T07:30:25.768 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:25 smithi083 ceph-mon[23711]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 481 op/s 2024-09-18T07:30:25.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:25 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:25.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:25 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:25.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:25 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:25.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:25 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:26.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:25 smithi096 ceph-mon[25928]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 481 op/s 2024-09-18T07:30:26.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:25 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:26.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:25 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:26.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:25 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:26.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:25 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:26.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 115223 10.68 MB/sec execute 155 sec latency 46.058 ms 2024-09-18T07:30:27.045 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:26 smithi083 ceph-mon[23711]: pgmap v22: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 663 op/s 2024-09-18T07:30:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:26 smithi096 ceph-mon[25928]: pgmap v22: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 663 op/s 2024-09-18T07:30:27.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 115614 10.68 MB/sec execute 156 sec latency 57.761 ms 2024-09-18T07:30:28.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:27 smithi083 ceph-mon[23711]: Standby manager daemon smithi096.caifnv restarted 2024-09-18T07:30:28.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:27 smithi083 ceph-mon[23711]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:30:28.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:27 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/907748960' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/crt"}]: dispatch 2024-09-18T07:30:28.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:27 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/907748960' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:30:28.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:27 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/907748960' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/key"}]: dispatch 2024-09-18T07:30:28.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:27 smithi083 ceph-mon[23711]: from='mgr.? 172.21.15.96:0/907748960' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:30:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:27 smithi096 ceph-mon[25928]: Standby manager daemon smithi096.caifnv restarted 2024-09-18T07:30:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:27 smithi096 ceph-mon[25928]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:30:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:27 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/907748960' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/crt"}]: dispatch 2024-09-18T07:30:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:27 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/907748960' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:30:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:27 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/907748960' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/key"}]: dispatch 2024-09-18T07:30:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:27 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/907748960' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:30:28.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 116043 10.66 MB/sec execute 157 sec latency 43.726 ms 2024-09-18T07:30:29.072 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:28 smithi083 ceph-mon[23711]: mgrmap e35: smithi083.vchvnu(active, since 40s), standbys: smithi096.caifnv 2024-09-18T07:30:29.073 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:28 smithi083 ceph-mon[23711]: pgmap v23: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-09-18T07:30:29.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:28 smithi096 ceph-mon[25928]: mgrmap e35: smithi083.vchvnu(active, since 40s), standbys: smithi096.caifnv 2024-09-18T07:30:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:28 smithi096 ceph-mon[25928]: pgmap v23: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-09-18T07:30:29.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 116397 10.65 MB/sec execute 158 sec latency 68.757 ms 2024-09-18T07:30:30.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:30:30.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:30:30.356 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.vchvnu"}]: dispatch 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.vchvnu"}]: dispatch 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.vchvnu"}]': finished 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi096.caifnv"}]: dispatch 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi096.caifnv"}]: dispatch 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi096.caifnv"}]': finished 2024-09-18T07:30:30.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:30 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-18T07:30:30.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 116804 10.68 MB/sec execute 159 sec latency 40.617 ms 2024-09-18T07:30:30.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:30:30.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "versions"}]: dispatch 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.vchvnu"}]: dispatch 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.vchvnu"}]: dispatch 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi083.vchvnu"}]': finished 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi096.caifnv"}]: dispatch 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi096.caifnv"}]: dispatch 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi096.caifnv"}]': finished 2024-09-18T07:30:30.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:30 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-18T07:30:31.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 117292 10.69 MB/sec execute 160 sec latency 91.906 ms 2024-09-18T07:30:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:31 smithi083 ceph-mon[23711]: Upgrade: Setting container_image for all mgr 2024-09-18T07:30:31.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:31 smithi083 ceph-mon[23711]: pgmap v24: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-18T07:30:31.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:31 smithi096 ceph-mon[25928]: Upgrade: Setting container_image for all mgr 2024-09-18T07:30:31.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:31 smithi096 ceph-mon[25928]: pgmap v24: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-18T07:30:32.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 117655 10.68 MB/sec execute 161 sec latency 33.729 ms 2024-09-18T07:30:33.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:33 smithi083 ceph-mon[23711]: pgmap v25: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-18T07:30:33.393 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:33 smithi083 ceph-mon[23711]: Upgrade: Updating mon.smithi083 2024-09-18T07:30:33.394 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:33 smithi083 ceph-mon[23711]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:33.394 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:33 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-18T07:30:33.394 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:33 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-18T07:30:33.394 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:33 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:30:33.394 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:33 smithi083 ceph-mon[23711]: Deploying daemon mon.smithi083 on smithi083 2024-09-18T07:30:33.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 118006 10.69 MB/sec execute 162 sec latency 54.099 ms 2024-09-18T07:30:33.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:33 smithi096 ceph-mon[25928]: pgmap v25: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-18T07:30:33.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:33 smithi096 ceph-mon[25928]: Upgrade: Updating mon.smithi083 2024-09-18T07:30:33.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:33 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:33.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:33 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-18T07:30:33.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:33 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-18T07:30:33.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:33 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-18T07:30:33.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:33 smithi096 ceph-mon[25928]: Deploying daemon mon.smithi083 on smithi083 2024-09-18T07:30:34.429 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:34 smithi083 ceph-mon[23711]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:30:34.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 118346 10.68 MB/sec execute 163 sec latency 48.557 ms 2024-09-18T07:30:34.577 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:34 smithi096 ceph-mon[25928]: from='mgr.14794 172.21.15.83:0/4049046720' entity='mgr.smithi083.vchvnu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-18T07:30:34.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:34 smithi083 systemd[1]: Stopping Ceph mon.smithi083 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:30:35.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:35 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083[23707]: 2024-09-18T07:30:35.090+0000 7fbc8ff44700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi083 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:30:35.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:35 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083[23707]: 2024-09-18T07:30:35.090+0000 7fbc8ff44700 -1 mon.smithi083@0(leader) e2 *** Got Signal Terminated *** 2024-09-18T07:30:35.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:35 smithi083 podman[63556]: 2024-09-18 07:30:35.141391904 +0000 UTC m=+0.216779302 container died 9f77aab8b41c29566ed04a6ad5d13afde2a31c1b8335c525b209784b131bddab (image=quay.io/ceph/ceph:v17.2.0, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., ceph=True, vcs-type=git, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, release=754, version=8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-18T07:30:35.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 118681 10.67 MB/sec execute 164 sec latency 54.653 ms 2024-09-18T07:30:35.987 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:35 smithi083 podman[63556]: 2024-09-18 07:30:35.984516574 +0000 UTC m=+1.059903971 container cleanup 9f77aab8b41c29566ed04a6ad5d13afde2a31c1b8335c525b209784b131bddab (image=quay.io/ceph/ceph:v17.2.0, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, architecture=x86_64, vcs-type=git, com.redhat.component=centos-stream-container, version=8, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., distribution-scope=public, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD) 2024-09-18T07:30:35.987 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:35 smithi083 bash[63556]: ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083 2024-09-18T07:30:36.243 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:36 smithi083 podman[63567]: 2024-09-18 07:30:36.200107413 +0000 UTC m=+1.050343386 container remove 9f77aab8b41c29566ed04a6ad5d13afde2a31c1b8335c525b209784b131bddab (image=quay.io/ceph/ceph:v17.2.0, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, name=centos-stream, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, release=754, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, maintainer=Guillaume Abrioux ) 2024-09-18T07:30:36.526 INFO:teuthology.orchestra.run.smithi083.stdout: 5 118998 10.65 MB/sec execute 165 sec latency 44.376 ms 2024-09-18T07:30:36.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:36 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi083.service: Deactivated successfully. 2024-09-18T07:30:36.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:36 smithi083 systemd[1]: Stopped Ceph mon.smithi083 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:30:36.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:36 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi083.service: Consumed 15.367s CPU time. 2024-09-18T07:30:37.008 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:36 smithi083 systemd[1]: Starting Ceph mon.smithi083 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:30:37.266 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 podman[63691]: 2024-09-18 07:30:36.944279023 +0000 UTC m=+0.019344085 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:30:37.266 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 podman[63691]: 2024-09-18 07:30:37.066279196 +0000 UTC m=+0.141344250 container create c72d955a581d13f3548bd1aef89b01164c895775d5c406482bc9a89b94b7c7df (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:30:37.524 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 podman[63691]: 2024-09-18 07:30:37.422270273 +0000 UTC m=+0.497335342 container init c72d955a581d13f3548bd1aef89b01164c895775d5c406482bc9a89b94b7c7df (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-18T07:30:37.524 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 podman[63691]: 2024-09-18 07:30:37.430915701 +0000 UTC m=+0.505980762 container start c72d955a581d13f3548bd1aef89b01164c895775d5c406482bc9a89b94b7c7df (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=) 2024-09-18T07:30:37.525 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: set uid:gid to 167:167 (ceph:ceph) 2024-09-18T07:30:37.525 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev), process ceph-mon, pid 2 2024-09-18T07:30:37.525 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: pidfile_write: ignore empty --pid-file 2024-09-18T07:30:37.525 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: load: jerasure load: lrc 2024-09-18T07:30:37.525 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: RocksDB version: 7.9.2 2024-09-18T07:30:37.525 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Git sha 0 2024-09-18T07:30:37.525 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Compile date 2024-09-17 15:59:30 2024-09-18T07:30:37.526 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: DB SUMMARY 2024-09-18T07:30:37.526 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: DB Session ID: AOVC6SGZIDFOME5SVZ22 2024-09-18T07:30:37.526 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: CURRENT file: CURRENT 2024-09-18T07:30:37.526 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: IDENTITY file: IDENTITY 2024-09-18T07:30:37.526 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-09-18T07:30:37.526 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi083/store.db dir, Total Num: 1, files: 000030.sst 2024-09-18T07:30:37.526 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi083/store.db: 000028.log size: 2974836 ; 2024-09-18T07:30:37.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.error_if_exists: 0 2024-09-18T07:30:37.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.create_if_missing: 0 2024-09-18T07:30:37.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.paranoid_checks: 1 2024-09-18T07:30:37.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-18T07:30:37.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-18T07:30:37.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-18T07:30:37.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.env: 0x55fd62137120 2024-09-18T07:30:37.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.fs: PosixFileSystem 2024-09-18T07:30:37.527 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.info_log: 0x55fd63aff8e0 2024-09-18T07:30:37.528 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_file_opening_threads: 16 2024-09-18T07:30:37.529 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.statistics: (nil) 2024-09-18T07:30:37.529 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.use_fsync: 0 2024-09-18T07:30:37.529 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_log_file_size: 0 2024-09-18T07:30:37.529 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-18T07:30:37.529 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-18T07:30:37.529 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.keep_log_file_num: 1000 2024-09-18T07:30:37.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.recycle_log_file_num: 0 2024-09-18T07:30:37.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.allow_fallocate: 1 2024-09-18T07:30:37.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.allow_mmap_reads: 0 2024-09-18T07:30:37.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.allow_mmap_writes: 0 2024-09-18T07:30:37.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.use_direct_reads: 0 2024-09-18T07:30:37.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-18T07:30:37.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.create_missing_column_families: 0 2024-09-18T07:30:37.530 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.db_log_dir: 2024-09-18T07:30:37.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.wal_dir: 2024-09-18T07:30:37.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-18T07:30:37.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-18T07:30:37.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-18T07:30:37.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-18T07:30:37.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-18T07:30:37.531 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.advise_random_on_open: 1 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.db_write_buffer_size: 0 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.write_buffer_manager: 0x55fd63b03b80 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.rate_limiter: (nil) 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.wal_recovery_mode: 2 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.enable_thread_tracking: 0 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.enable_pipelined_write: 0 2024-09-18T07:30:37.532 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.unordered_write: 0 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.row_cache: None 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.wal_filter: None 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.allow_ingest_behind: 0 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.two_write_queues: 0 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.manual_wal_flush: 0 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.wal_compression: 0 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.atomic_flush: 0 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-18T07:30:37.533 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.log_readahead_size: 0 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.best_efforts_recovery: 0 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.allow_data_in_errors: 0 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.db_host_id: __hostname__ 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_background_jobs: 2 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_background_compactions: -1 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_subcompactions: 1 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-18T07:30:37.534 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_total_wal_size: 0 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_open_files: -1 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bytes_per_sync: 0 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_readahead_size: 0 2024-09-18T07:30:37.535 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_background_flushes: -1 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Compression algorithms supported: 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: kZSTD supported: 0 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: kXpressCompression supported: 0 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: kBZip2Compression supported: 0 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: kLZ4Compression supported: 1 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: kZlibCompression supported: 1 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: kLZ4HCCompression supported: 1 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: kSnappyCompression supported: 1 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000015 2024-09-18T07:30:37.536 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.merge_operator: 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_filter: None 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_filter_factory: None 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.sst_partitioner_factory: None 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fd63aff540) 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks: 1 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: pin_top_level_index_and_filter: 1 2024-09-18T07:30:37.537 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_type: 0 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_index_type: 0 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_shortening: 1 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: checksum: 4 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: no_block_cache: 0 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache: 0x55fd63af3610 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_name: BinnedLRUCache 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_options: 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: capacity : 536870912 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: num_shard_bits : 4 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: strict_capacity_limit : 0 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: high_pri_pool_ratio: 0.000 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_cache_compressed: (nil) 2024-09-18T07:30:37.538 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: persistent_cache: (nil) 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size: 4096 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_size_deviation: 10 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_restart_interval: 16 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: index_block_restart_interval: 1 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: metadata_block_size: 4096 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: partition_filters: 0 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: use_delta_encoding: 1 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: filter_policy: bloomfilter 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: whole_key_filtering: 1 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: verify_compression: 0 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: read_amp_bytes_per_bit: 0 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: format_version: 5 2024-09-18T07:30:37.539 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_index_compression: 1 2024-09-18T07:30:37.540 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: block_align: 0 2024-09-18T07:30:37.540 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: max_auto_readahead_size: 262144 2024-09-18T07:30:37.540 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: prepopulate_block_cache: 0 2024-09-18T07:30:37.540 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: initial_auto_readahead_size: 8192 2024-09-18T07:30:37.540 INFO:teuthology.orchestra.run.smithi083.stdout: 5 119413 10.64 MB/sec execute 166 sec latency 53.096 ms 2024-09-18T07:30:37.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: num_file_reads_for_auto_readahead: 2 2024-09-18T07:30:37.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.write_buffer_size: 33554432 2024-09-18T07:30:37.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_write_buffer_number: 2 2024-09-18T07:30:37.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression: NoCompression 2024-09-18T07:30:37.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression: Disabled 2024-09-18T07:30:37.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.prefix_extractor: nullptr 2024-09-18T07:30:37.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-18T07:30:37.541 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.num_levels: 7 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression_opts.level: 32767 2024-09-18T07:30:37.542 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression_opts.strategy: 0 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression_opts.enabled: false 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.target_file_size_base: 67108864 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-18T07:30:37.543 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.arena_block_size: 1048576 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.disable_auto_compactions: 0 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-18T07:30:37.544 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.inplace_update_support: 0 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-18T07:30:37.545 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.bloom_locality: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.max_successive_merges: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.paranoid_file_checks: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.force_consistency_checks: 1 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.report_bg_io_stats: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.ttl: 2592000 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.enable_blob_files: false 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.min_blob_size: 0 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.blob_file_size: 268435456 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-18T07:30:37.546 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.blob_file_starting_level: 0 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi083/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11858, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 2494b820-adfd-473a-a9f1-d827faedd810 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726644637463097, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-18T07:30:37.547 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726644637472734, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2743625, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11852, "largest_seqno": 12949, "table_properties": {"data_size": 2738071, "index_size": 3293, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10890, "raw_average_key_size": 23, "raw_value_size": 2727758, "raw_average_value_size": 5766, "num_data_blocks": 150, "num_entries": 473, "num_filter_entries": 473, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726644637, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "2494b820-adfd-473a-a9f1-d827faedd810", "db_session_id": "AOVC6SGZIDFOME5SVZ22", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726644637472903, "job": 1, "event": "recovery_finished"} 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi083/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55fd63b22e00 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: DB pointer 0x55fd63c24000 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** DB Stats ** 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-18T07:30:37.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L0 1/0 2.62 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: L6 1/0 7.03 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Sum 2/0 9.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:30:37.549 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** Compaction Stats [default] ** 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 487.4 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Cumulative compaction: 0.00 GB write, 153.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-18T07:30:37.550 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Interval compaction: 0.00 GB write, 153.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Block cache BinnedLRUCache@0x55fd63af3610#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: starting mon.smithi083 rank 0 at public addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] at bind addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi083 fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: mon.smithi083@-1(???) e2 preinit fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: mon.smithi083@-1(???).mds e7 new map 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: mon.smithi083@-1(???).mds e7 print_map 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: e7 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-18T07:30:37.551 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: legacy client fscid: 1 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Filesystem 'foofs' (1) 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: fs_name foofs 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: epoch 5 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: created 2024-09-18T07:25:14.675748+0000 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: modified 2024-09-18T07:25:20.697480+0000 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: tableserver 0 2024-09-18T07:30:37.552 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: root 0 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: session_timeout 60 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: session_autoclose 300 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: max_file_size 1099511627776 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: max_xattr_size 65536 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: required_client_features {} 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: last_failure 0 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: last_failure_osd_epoch 0 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: max_mds 1 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: in 0 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: up {0=14502} 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: failed 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: damaged 2024-09-18T07:30:37.553 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: stopped 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: data_pools [3] 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: metadata_pool 2 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: inline_data disabled 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: balancer 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: bal_rank_mask -1 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: standby_count_wanted 1 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: qdb_cluster leader: 0 members: 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: [mds.foofs.smithi083.kjbbry{0:14502} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.83:6834/2180084897,v1:172.21.15.83:6835/2180084897] compat {c=[1],r=[1],i=[7ff]}] 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Standby daemons: 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: [mds.foofs.smithi096.bzilgv{-1:24293} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] compat {c=[1],r=[1],i=[7ff]}] 2024-09-18T07:30:37.554 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: mon.smithi083@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-18T07:30:37.555 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: mon.smithi083@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-18T07:30:37.555 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: mon.smithi083@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-18T07:30:37.555 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: mon.smithi083@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-18T07:30:37.555 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 ceph-mon[63709]: mon.smithi083@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-18T07:30:37.555 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 bash[63691]: c72d955a581d13f3548bd1aef89b01164c895775d5c406482bc9a89b94b7c7df 2024-09-18T07:30:37.555 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:37 smithi083 systemd[1]: Started Ceph mon.smithi083 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:30:38.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 119802 10.63 MB/sec execute 167 sec latency 35.928 ms 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: pgmap v27: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 657 op/s 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: pgmap v28: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: mon.smithi083 calling monitor election 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: mon.smithi083 is new leader, mons smithi083,smithi096 in quorum (ranks 0,1) 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: monmap epoch 2 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: last_changed 2024-09-18T07:20:05.813849+0000 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: created 2024-09-18T07:17:52.460906+0000 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: min_mon_release 17 (quincy) 2024-09-18T07:30:39.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: election_strategy: 1 2024-09-18T07:30:39.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: 0: [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon.smithi083 2024-09-18T07:30:39.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: 1: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.smithi096 2024-09-18T07:30:39.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:30:39.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: osdmap e72: 8 total, 8 up, 8 in 2024-09-18T07:30:39.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: mgrmap e35: smithi083.vchvnu(active, since 51s), standbys: smithi096.caifnv 2024-09-18T07:30:39.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: overall HEALTH_OK 2024-09-18T07:30:39.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: from='mgr.14794 ' entity='' 2024-09-18T07:30:39.058 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:38 smithi083 ceph-mon[63709]: mgrmap e36: smithi083.vchvnu(active, since 51s), standbys: smithi096.caifnv 2024-09-18T07:30:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: pgmap v27: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 657 op/s 2024-09-18T07:30:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: pgmap v28: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-18T07:30:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: mon.smithi083 calling monitor election 2024-09-18T07:30:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: mon.smithi083 is new leader, mons smithi083,smithi096 in quorum (ranks 0,1) 2024-09-18T07:30:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: monmap epoch 2 2024-09-18T07:30:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:30:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: last_changed 2024-09-18T07:20:05.813849+0000 2024-09-18T07:30:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: created 2024-09-18T07:17:52.460906+0000 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: min_mon_release 17 (quincy) 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: election_strategy: 1 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: 0: [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon.smithi083 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: 1: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.smithi096 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: osdmap e72: 8 total, 8 up, 8 in 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: mgrmap e35: smithi083.vchvnu(active, since 51s), standbys: smithi096.caifnv 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: overall HEALTH_OK 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: from='mgr.14794 ' entity='' 2024-09-18T07:30:39.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:38 smithi096 ceph-mon[25928]: mgrmap e36: smithi083.vchvnu(active, since 51s), standbys: smithi096.caifnv 2024-09-18T07:30:39.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 120214 10.65 MB/sec execute 168 sec latency 49.397 ms 2024-09-18T07:30:40.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 120734 10.68 MB/sec execute 169 sec latency 50.584 ms 2024-09-18T07:30:41.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:40 smithi083 ceph-mon[63709]: mgrmap e37: smithi083.vchvnu(active, since 52s), standbys: smithi096.caifnv 2024-09-18T07:30:41.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:40 smithi096 ceph-mon[25928]: mgrmap e37: smithi083.vchvnu(active, since 52s), standbys: smithi096.caifnv 2024-09-18T07:30:41.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 121134 10.67 MB/sec execute 170 sec latency 54.198 ms 2024-09-18T07:30:42.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 121364 10.65 MB/sec execute 171 sec latency 114.967 ms 2024-09-18T07:30:43.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 121712 10.66 MB/sec execute 172 sec latency 57.736 ms 2024-09-18T07:30:44.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 122028 10.63 MB/sec execute 173 sec latency 55.677 ms 2024-09-18T07:30:45.439 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:45 smithi083 ceph-mon[63709]: Standby manager daemon smithi096.caifnv restarted 2024-09-18T07:30:45.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:45 smithi083 ceph-mon[63709]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:30:45.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:45 smithi083 ceph-mon[63709]: from='mgr.? 172.21.15.96:0/414021433' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/crt"}]: dispatch 2024-09-18T07:30:45.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:45 smithi083 ceph-mon[63709]: from='mgr.? 172.21.15.96:0/414021433' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:30:45.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:45 smithi083 ceph-mon[63709]: from='mgr.? 172.21.15.96:0/414021433' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/key"}]: dispatch 2024-09-18T07:30:45.440 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:45 smithi083 ceph-mon[63709]: from='mgr.? 172.21.15.96:0/414021433' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:30:45.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 122310 10.61 MB/sec execute 174 sec latency 53.275 ms 2024-09-18T07:30:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:45 smithi096 ceph-mon[25928]: Standby manager daemon smithi096.caifnv restarted 2024-09-18T07:30:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:45 smithi096 ceph-mon[25928]: Standby manager daemon smithi096.caifnv started 2024-09-18T07:30:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:45 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/414021433' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/crt"}]: dispatch 2024-09-18T07:30:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:45 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/414021433' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-18T07:30:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:45 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/414021433' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi096.caifnv/key"}]: dispatch 2024-09-18T07:30:45.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:45 smithi096 ceph-mon[25928]: from='mgr.? 172.21.15.96:0/414021433' entity='mgr.smithi096.caifnv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-18T07:30:46.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: mgrmap e38: smithi083.vchvnu(active, since 57s), standbys: smithi096.caifnv 2024-09-18T07:30:46.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: Active manager daemon smithi083.vchvnu restarted 2024-09-18T07:30:46.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: Activating manager daemon smithi083.vchvnu 2024-09-18T07:30:46.462 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: osdmap e73: 8 total, 8 up, 8 in 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: mgrmap e39: smithi083.vchvnu(active, starting, since 0.0113829s), standbys: smithi096.caifnv 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata", "id": "smithi083"} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata", "id": "smithi096"} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mgr metadata", "who": "smithi096.caifnv", "id": "smithi096.caifnv"} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-18T07:30:46.463 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata"} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata"} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata"} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: Manager daemon smithi083.vchvnu is now available 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"} : dispatch 2024-09-18T07:30:46.464 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"} : dispatch 2024-09-18T07:30:46.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 122516 10.59 MB/sec execute 175 sec latency 63.555 ms 2024-09-18T07:30:46.573 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:30:46.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: mgrmap e38: smithi083.vchvnu(active, since 57s), standbys: smithi096.caifnv 2024-09-18T07:30:46.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: Active manager daemon smithi083.vchvnu restarted 2024-09-18T07:30:46.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: Activating manager daemon smithi083.vchvnu 2024-09-18T07:30:46.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: osdmap e73: 8 total, 8 up, 8 in 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: mgrmap e39: smithi083.vchvnu(active, starting, since 0.0113829s), standbys: smithi096.caifnv 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata", "id": "smithi083"} : dispatch 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata", "id": "smithi096"} : dispatch 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"} : dispatch 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"} : dispatch 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mgr metadata", "who": "smithi083.vchvnu", "id": "smithi083.vchvnu"} : dispatch 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mgr metadata", "who": "smithi096.caifnv", "id": "smithi096.caifnv"} : dispatch 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-18T07:30:46.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata"} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata"} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata"} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: Manager daemon smithi083.vchvnu is now available 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:30:46.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-18T07:30:46.625 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/mirror_snapshot_schedule"} : dispatch 2024-09-18T07:30:46.625 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:46 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi083.vchvnu/trash_purge_schedule"} : dispatch 2024-09-18T07:30:47.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 122780 10.57 MB/sec execute 176 sec latency 58.767 ms 2024-09-18T07:30:47.589 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:47 smithi083 ceph-mon[63709]: mgrmap e40: smithi083.vchvnu(active, since 1.0124s), standbys: smithi096.caifnv 2024-09-18T07:30:47.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:47 smithi083 ceph-mon[63709]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:47.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:47 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:30:47.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:47 smithi083 ceph-mon[63709]: pgmap v3: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-18T07:30:47.740 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:30:47.740 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (2m) 18s ago 11m 19.8M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:30:47.740 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (11m) 18s ago 11m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:30:47.740 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (10m) 22s ago 10m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:30:47.740 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (2m) 18s ago 11m 46.8M - dad864ee21e9 06ff04a18f50 2024-09-18T07:30:47.740 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (5m) 18s ago 5m 40.0M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:30:47.740 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (5m) 22s ago 5m 15.3M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:30:47.740 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (76s) 18s ago 12m 508M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:30:47.740 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (27s) 22s ago 10m 67.5M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (12m) 18s ago 12m 55.9M 2048M 17.2.0 e1d6a67b021e 9f77aab8b41c 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (10m) 22s ago 10m 45.4M 2048M 17.2.0 e1d6a67b021e 10af61cef71b 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (5m) 18s ago 5m 269M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (5m) 22s ago 5m 87.7M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (113s) 18s ago 11m 10.3M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (85s) 22s ago 10m 8002k - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (9m) 18s ago 9m 455M 1936M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9m) 18s ago 9m 367M 1936M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8m) 18s ago 8m 279M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (8m) 18s ago 8m 411M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (8m) 22s ago 8m 398M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (7m) 22s ago 7m 434M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:30:47.741 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (7m) 22s ago 7m 376M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:30:47.742 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (6m) 22s ago 6m 526M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:30:47.742 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (33s) 18s ago 10m 43.7M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:30:47.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:47 smithi096 ceph-mon[25928]: mgrmap e40: smithi083.vchvnu(active, since 1.0124s), standbys: smithi096.caifnv 2024-09-18T07:30:47.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:47 smithi096 ceph-mon[25928]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:47.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:47 smithi096 ceph-mon[25928]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-18T07:30:47.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:47 smithi096 ceph-mon[25928]: pgmap v3: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-18T07:30:48.422 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:30:48.422 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:30:48.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-18T07:30:48.422 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 1 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 3 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:30:48.423 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:30:48.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 123100 10.53 MB/sec execute 177 sec latency 79.130 ms 2024-09-18T07:30:48.604 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:48 smithi083 ceph-mon[63709]: [18/Sep/2024:07:30:47] ENGINE Bus STARTING 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:48 smithi083 ceph-mon[63709]: [18/Sep/2024:07:30:47] ENGINE Serving on https://172.21.15.83:7150 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:48 smithi083 ceph-mon[63709]: [18/Sep/2024:07:30:47] ENGINE Error in HTTPServer.serve 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: Traceback (most recent call last): 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self._connections.run(self.expiration_interval) 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self._run(expiration_interval) 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-18T07:30:48.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: s = self.context.wrap_socket( 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: return self.sslsocket_class._create( 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self.do_handshake() 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: self._sslobj.do_handshake() 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:48 smithi083 ceph-mon[63709]: [18/Sep/2024:07:30:47] ENGINE Serving on http://172.21.15.83:8765 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:48 smithi083 ceph-mon[63709]: [18/Sep/2024:07:30:47] ENGINE Bus STARTED 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:48 smithi083 ceph-mon[63709]: pgmap v4: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:48 smithi083 ceph-mon[63709]: mgrmap e41: smithi083.vchvnu(active, since 2s), standbys: smithi096.caifnv 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:48 smithi083 ceph-mon[63709]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:48.606 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:48 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/3266342823' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:48 smithi096 ceph-mon[25928]: [18/Sep/2024:07:30:47] ENGINE Bus STARTING 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:48 smithi096 ceph-mon[25928]: [18/Sep/2024:07:30:47] ENGINE Serving on https://172.21.15.83:7150 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:48 smithi096 ceph-mon[25928]: [18/Sep/2024:07:30:47] ENGINE Error in HTTPServer.serve 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Traceback (most recent call last): 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self._connections.run(self.expiration_interval) 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self._run(expiration_interval) 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-18T07:30:48.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: s = self.context.wrap_socket( 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: return self.sslsocket_class._create( 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self.do_handshake() 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: self._sslobj.do_handshake() 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:48 smithi096 ceph-mon[25928]: [18/Sep/2024:07:30:47] ENGINE Serving on http://172.21.15.83:8765 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:48 smithi096 ceph-mon[25928]: [18/Sep/2024:07:30:47] ENGINE Bus STARTED 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:48 smithi096 ceph-mon[25928]: pgmap v4: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:48 smithi096 ceph-mon[25928]: mgrmap e41: smithi083.vchvnu(active, since 2s), standbys: smithi096.caifnv 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:48 smithi096 ceph-mon[25928]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:48.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:48 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/3266342823' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:30:48.936 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:30:48.936 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:30:48.936 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:30:48.936 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:30:48.936 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:30:48.936 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr" 2024-09-18T07:30:48.936 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:30:48.936 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "2/23 daemons upgraded", 2024-09-18T07:30:48.937 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-09-18T07:30:48.937 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:30:48.937 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:30:49.481 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:30:49.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 123369 10.51 MB/sec execute 178 sec latency 69.281 ms 2024-09-18T07:30:49.626 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:49 smithi083 ceph-mon[63709]: from='client.24625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:49.626 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:49 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/597740120' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:30:49.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:49 smithi096 ceph-mon[25928]: from='client.24625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:30:49.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:49 smithi096 ceph-mon[25928]: from='client.? 172.21.15.83:0/597740120' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:30:50.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 123666 10.52 MB/sec execute 179 sec latency 50.599 ms 2024-09-18T07:30:50.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:50 smithi083 ceph-mon[63709]: pgmap v5: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-18T07:30:50.639 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:50 smithi083 ceph-mon[63709]: mgrmap e42: smithi083.vchvnu(active, since 4s), standbys: smithi096.caifnv 2024-09-18T07:30:50.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:50 smithi096 ceph-mon[25928]: pgmap v5: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-18T07:30:50.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:50 smithi096 ceph-mon[25928]: mgrmap e42: smithi083.vchvnu(active, since 4s), standbys: smithi096.caifnv 2024-09-18T07:30:51.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 124006 10.52 MB/sec execute 180 sec latency 86.153 ms 2024-09-18T07:30:51.933 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:51.933 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:51.934 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:51.934 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:52.075 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:51 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:52.075 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:51 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:52.075 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:51 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:52.075 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:51 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:52.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 124349 10.51 MB/sec execute 181 sec latency 65.388 ms 2024-09-18T07:30:52.978 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:52 smithi083 ceph-mon[63709]: pgmap v6: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 458 KiB/s wr, 17 op/s 2024-09-18T07:30:53.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:52 smithi096 ceph-mon[25928]: pgmap v6: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 458 KiB/s wr, 17 op/s 2024-09-18T07:30:53.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 124719 10.50 MB/sec execute 182 sec latency 48.257 ms 2024-09-18T07:30:54.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 124977 10.48 MB/sec execute 183 sec latency 43.580 ms 2024-09-18T07:30:55.018 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:54 smithi083 ceph-mon[63709]: pgmap v7: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 2.9 MiB/s wr, 312 op/s 2024-09-18T07:30:55.018 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:55.018 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:55.085 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:54 smithi096 ceph-mon[25928]: pgmap v7: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 2.9 MiB/s wr, 312 op/s 2024-09-18T07:30:55.086 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:54 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:55.086 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:54 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:55.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 125313 10.48 MB/sec execute 184 sec latency 41.891 ms 2024-09-18T07:30:56.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:30:56.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:30:56.325 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: Updating smithi083:/etc/ceph/ceph.conf 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: pgmap v8: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 243 op/s 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.326 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 125665 10.46 MB/sec execute 185 sec latency 61.505 ms 2024-09-18T07:30:56.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:30:56.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:30:56.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: Updating smithi083:/etc/ceph/ceph.conf 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: Updating smithi096:/etc/ceph/ceph.conf 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: pgmap v8: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 243 op/s 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: Updating smithi096:/etc/ceph/ceph.client.admin.keyring 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: Updating smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: Updating smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:56.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:56 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:57.338 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:57 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:30:57.338 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:57 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:30:57.339 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:57 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "quorum_status"} : dispatch 2024-09-18T07:30:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:57 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:30:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:57 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:30:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:57 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "quorum_status"} : dispatch 2024-09-18T07:30:57.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 125925 10.46 MB/sec execute 186 sec latency 54.554 ms 2024-09-18T07:30:58.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:58 smithi083 ceph-mon[63709]: pgmap v9: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-09-18T07:30:58.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 126142 10.43 MB/sec execute 187 sec latency 84.655 ms 2024-09-18T07:30:58.598 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:58 smithi096 ceph-mon[25928]: pgmap v9: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-09-18T07:30:59.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 126429 10.42 MB/sec execute 188 sec latency 77.188 ms 2024-09-18T07:30:59.875 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:59 smithi083 ceph-mon[63709]: Upgrade: Updating mon.smithi096 2024-09-18T07:30:59.875 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:59 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:30:59.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:59 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-18T07:30:59.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:59 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-18T07:30:59.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:59 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:30:59.876 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:30:59 smithi083 ceph-mon[63709]: Deploying daemon mon.smithi096 on smithi096 2024-09-18T07:31:00.061 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:59 smithi096 ceph-mon[25928]: Upgrade: Updating mon.smithi096 2024-09-18T07:31:00.061 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:59 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:00.061 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:59 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-18T07:31:00.061 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:59 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-18T07:31:00.061 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:59 smithi096 ceph-mon[25928]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:00.061 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:30:59 smithi096 ceph-mon[25928]: Deploying daemon mon.smithi096 on smithi096 2024-09-18T07:31:00.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 126770 10.38 MB/sec execute 189 sec latency 61.249 ms 2024-09-18T07:31:00.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:00 smithi096 systemd[1]: Stopping Ceph mon.smithi096 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:31:00.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:00 smithi083 ceph-mon[63709]: pgmap v10: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.7 MiB/s wr, 376 op/s 2024-09-18T07:31:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:00 smithi096 ceph-mon[25928]: pgmap v10: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.7 MiB/s wr, 376 op/s 2024-09-18T07:31:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096[25922]: 2024-09-18T07:31:00.657+0000 7facb5130700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi096 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:31:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096[25922]: 2024-09-18T07:31:00.657+0000 7facb5130700 -1 mon.smithi096@1(peon) e2 *** Got Signal Terminated *** 2024-09-18T07:31:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:00 smithi096 podman[49328]: 2024-09-18 07:31:00.70904685 +0000 UTC m=+0.180159080 container died 10af61cef71b1557a68497d4732f44df10e2e8c5eb26bb4c4d0d4c376f8b6c5e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096, vcs-type=git, name=centos-stream, GIT_CLEAN=True, distribution-scope=public, release=754, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux ) 2024-09-18T07:31:01.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 127142 10.40 MB/sec execute 190 sec latency 39.020 ms 2024-09-18T07:31:01.858 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:01 smithi096 podman[49328]: 2024-09-18 07:31:01.589021437 +0000 UTC m=+1.060133656 container cleanup 10af61cef71b1557a68497d4732f44df10e2e8c5eb26bb4c4d0d4c376f8b6c5e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_CLEAN=True, vcs-type=git, vendor=Red Hat, Inc., io.openshift.expose-services=, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, version=8, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0) 2024-09-18T07:31:01.858 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:01 smithi096 bash[49328]: ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096 2024-09-18T07:31:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:01 smithi096 podman[49341]: 2024-09-18 07:31:01.856080662 +0000 UTC m=+1.148456927 container remove 10af61cef71b1557a68497d4732f44df10e2e8c5eb26bb4c4d0d4c376f8b6c5e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, vendor=Red Hat, Inc., version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, ceph=True, architecture=x86_64, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vcs-type=git) 2024-09-18T07:31:02.172 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi096.service: Deactivated successfully. 2024-09-18T07:31:02.173 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 systemd[1]: Stopped Ceph mon.smithi096 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:31:02.173 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi096.service: Consumed 6.982s CPU time. 2024-09-18T07:31:02.527 INFO:teuthology.orchestra.run.smithi083.stdout: 5 127582 10.41 MB/sec execute 191 sec latency 40.290 ms 2024-09-18T07:31:02.584 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 systemd[1]: Starting Ceph mon.smithi096 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:31:02.584 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 podman[49465]: 2024-09-18 07:31:02.461279299 +0000 UTC m=+0.019052045 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:31:02.854 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 podman[49465]: 2024-09-18 07:31:02.582801973 +0000 UTC m=+0.140574723 container create baa2fbd25f23ddc1711b739464db15e5ef88d4eb81c76625145790deb6af1d95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-18T07:31:02.855 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 podman[49465]: 2024-09-18 07:31:02.852824645 +0000 UTC m=+0.410597393 container init baa2fbd25f23ddc1711b739464db15e5ef88d4eb81c76625145790deb6af1d95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True) 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 podman[49465]: 2024-09-18 07:31:02.857213483 +0000 UTC m=+0.414986222 container start baa2fbd25f23ddc1711b739464db15e5ef88d4eb81c76625145790deb6af1d95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2) 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: set uid:gid to 167:167 (ceph:ceph) 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev), process ceph-mon, pid 2 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: pidfile_write: ignore empty --pid-file 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: load: jerasure load: lrc 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: RocksDB version: 7.9.2 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Git sha 0 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Compile date 2024-09-17 15:59:30 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: DB SUMMARY 2024-09-18T07:31:03.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: DB Session ID: O9H79OVN5WQV8H3ZXB9B 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: CURRENT file: CURRENT 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: IDENTITY file: IDENTITY 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi096/store.db dir, Total Num: 1, files: 000024.sst 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi096/store.db: 000022.log size: 7934495 ; 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.error_if_exists: 0 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.create_if_missing: 0 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.paranoid_checks: 1 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.env: 0x564d573f7120 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.fs: PosixFileSystem 2024-09-18T07:31:03.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.info_log: 0x564d57edb8e0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_file_opening_threads: 16 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.statistics: (nil) 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.use_fsync: 0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_log_file_size: 0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.keep_log_file_num: 1000 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.recycle_log_file_num: 0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.allow_fallocate: 1 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.allow_mmap_reads: 0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.allow_mmap_writes: 0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.use_direct_reads: 0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.create_missing_column_families: 0 2024-09-18T07:31:03.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.db_log_dir: 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.wal_dir: 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.advise_random_on_open: 1 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.db_write_buffer_size: 0 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.write_buffer_manager: 0x564d57edfb80 2024-09-18T07:31:03.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.rate_limiter: (nil) 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.wal_recovery_mode: 2 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.enable_thread_tracking: 0 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.enable_pipelined_write: 0 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.unordered_write: 0 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-18T07:31:03.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.row_cache: None 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.wal_filter: None 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.allow_ingest_behind: 0 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.two_write_queues: 0 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.manual_wal_flush: 0 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.wal_compression: 0 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.atomic_flush: 0 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-18T07:31:03.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-18T07:31:03.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-18T07:31:03.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.log_readahead_size: 0 2024-09-18T07:31:03.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-18T07:31:03.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.best_efforts_recovery: 0 2024-09-18T07:31:03.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-18T07:31:03.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-18T07:31:03.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.allow_data_in_errors: 0 2024-09-18T07:31:03.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.db_host_id: __hostname__ 2024-09-18T07:31:03.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_background_jobs: 2 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_background_compactions: -1 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_subcompactions: 1 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_total_wal_size: 0 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_open_files: -1 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bytes_per_sync: 0 2024-09-18T07:31:03.131 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_readahead_size: 0 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_background_flushes: -1 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Compression algorithms supported: 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: kZSTD supported: 0 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: kXpressCompression supported: 0 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: kBZip2Compression supported: 0 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: kLZ4Compression supported: 1 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: kZlibCompression supported: 1 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: kLZ4HCCompression supported: 1 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: kSnappyCompression supported: 1 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-18T07:31:03.132 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi096/store.db/MANIFEST-000009 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.merge_operator: 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_filter: None 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_filter_factory: None 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.sst_partitioner_factory: None 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564d57edb540) 2024-09-18T07:31:03.133 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: cache_index_and_filter_blocks: 1 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: pin_top_level_index_and_filter: 1 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: index_type: 0 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: data_block_index_type: 0 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: index_shortening: 1 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: checksum: 4 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: no_block_cache: 0 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: block_cache: 0x564d57ecf610 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: block_cache_name: BinnedLRUCache 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: block_cache_options: 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: capacity : 536870912 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: num_shard_bits : 4 2024-09-18T07:31:03.134 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: strict_capacity_limit : 0 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: high_pri_pool_ratio: 0.000 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: block_cache_compressed: (nil) 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: persistent_cache: (nil) 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: block_size: 4096 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: block_size_deviation: 10 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: block_restart_interval: 16 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: index_block_restart_interval: 1 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: metadata_block_size: 4096 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: partition_filters: 0 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: use_delta_encoding: 1 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: filter_policy: bloomfilter 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: whole_key_filtering: 1 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: verify_compression: 0 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: read_amp_bytes_per_bit: 0 2024-09-18T07:31:03.135 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: format_version: 5 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: enable_index_compression: 1 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: block_align: 0 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: max_auto_readahead_size: 262144 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: prepopulate_block_cache: 0 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: initial_auto_readahead_size: 8192 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: num_file_reads_for_auto_readahead: 2 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.write_buffer_size: 33554432 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_write_buffer_number: 2 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression: NoCompression 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression: Disabled 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.prefix_extractor: nullptr 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.num_levels: 7 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-18T07:31:03.136 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression_opts.level: 32767 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression_opts.strategy: 0 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-18T07:31:03.137 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression_opts.enabled: false 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.target_file_size_base: 67108864 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-18T07:31:03.138 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-18T07:31:03.139 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-18T07:31:03.139 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-18T07:31:03.139 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-18T07:31:03.139 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-18T07:31:03.139 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-18T07:31:03.139 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-18T07:31:03.139 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-18T07:31:03.139 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-18T07:31:03.139 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.arena_block_size: 1048576 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.disable_auto_compactions: 0 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-18T07:31:03.140 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.inplace_update_support: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.bloom_locality: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.max_successive_merges: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.paranoid_file_checks: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.force_consistency_checks: 1 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.report_bg_io_stats: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.ttl: 2592000 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.enable_blob_files: false 2024-09-18T07:31:03.141 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.min_blob_size: 0 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.blob_file_size: 268435456 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.blob_file_starting_level: 0 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi096/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12265, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7ebf5b04-abee-403c-bbd7-046e28378451 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726644662892787, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-18T07:31:03.142 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726644662913782, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4844720, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12262, "largest_seqno": 14002, "table_properties": {"data_size": 4836965, "index_size": 4918, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1797, "raw_key_size": 16038, "raw_average_key_size": 23, "raw_value_size": 4821905, "raw_average_value_size": 6947, "num_data_blocks": 223, "num_entries": 694, "num_filter_entries": 694, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726644662, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7ebf5b04-abee-403c-bbd7-046e28378451", "db_session_id": "O9H79OVN5WQV8H3ZXB9B", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726644662913933, "job": 1, "event": "recovery_finished"} 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi096/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564d57efee00 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: DB pointer 0x564d58000000 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: ** DB Stats ** 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-18T07:31:03.143 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: L0 1/0 4.62 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 508.1 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: L6 1/0 7.03 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Sum 2/0 11.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 508.1 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 508.1 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: 2024-09-18T07:31:03.144 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: ** Compaction Stats [default] ** 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 508.1 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Cumulative compaction: 0.00 GB write, 186.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-18T07:31:03.145 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Interval compaction: 0.00 GB write, 186.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Block cache BinnedLRUCache@0x564d57ecf610#2 capacity: 512.00 MB usage: 7.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: starting mon.smithi096 rank 1 at public addrs [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] at bind addrs [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi096 fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: mon.smithi096@-1(???) e2 preinit fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: mon.smithi096@-1(???).mds e7 new map 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: mon.smithi096@-1(???).mds e7 print_map 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: e7 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-18T07:31:03.146 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: legacy client fscid: 1 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Filesystem 'foofs' (1) 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: fs_name foofs 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: epoch 5 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: created 2024-09-18T07:25:14.675748+0000 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: modified 2024-09-18T07:25:20.697480+0000 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: tableserver 0 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: root 0 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: session_timeout 60 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: session_autoclose 300 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: max_file_size 1099511627776 2024-09-18T07:31:03.147 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: max_xattr_size 65536 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: required_client_features {} 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: last_failure 0 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: last_failure_osd_epoch 0 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: max_mds 1 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: in 0 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: up {0=14502} 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: failed 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: damaged 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: stopped 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: data_pools [3] 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: metadata_pool 2 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: inline_data disabled 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: balancer 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: bal_rank_mask -1 2024-09-18T07:31:03.148 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: standby_count_wanted 1 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: qdb_cluster leader: 0 members: 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: [mds.foofs.smithi083.kjbbry{0:14502} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.83:6834/2180084897,v1:172.21.15.83:6835/2180084897] compat {c=[1],r=[1],i=[7ff]}] 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: Standby daemons: 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout: [mds.foofs.smithi096.bzilgv{-1:24293} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] compat {c=[1],r=[1],i=[7ff]}] 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: mon.smithi096@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: mon.smithi096@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: mon.smithi096@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: mon.smithi096@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 ceph-mon[49483]: mon.smithi096@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 bash[49465]: baa2fbd25f23ddc1711b739464db15e5ef88d4eb81c76625145790deb6af1d95 2024-09-18T07:31:03.149 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:02 smithi096 systemd[1]: Started Ceph mon.smithi096 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:31:03.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 128205 10.44 MB/sec execute 192 sec latency 36.129 ms 2024-09-18T07:31:04.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 128675 10.46 MB/sec execute 193 sec latency 36.016 ms 2024-09-18T07:31:05.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: pgmap v12: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.4 MiB/s wr, 561 op/s 2024-09-18T07:31:05.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata", "id": "smithi083"} : dispatch 2024-09-18T07:31:05.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata", "id": "smithi096"} : dispatch 2024-09-18T07:31:05.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: mon.smithi096 calling monitor election 2024-09-18T07:31:05.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: mon.smithi083 calling monitor election 2024-09-18T07:31:05.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: mon.smithi083 is new leader, mons smithi083,smithi096 in quorum (ranks 0,1) 2024-09-18T07:31:05.002 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: monmap epoch 3 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: last_changed 2024-09-18T07:31:03.917446+0000 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: created 2024-09-18T07:17:52.460906+0000 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: min_mon_release 19 (squid) 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: election_strategy: 1 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: 0: [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon.smithi083 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: 1: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.smithi096 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: osdmap e73: 8 total, 8 up, 8 in 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: mgrmap e42: smithi083.vchvnu(active, since 18s), standbys: smithi096.caifnv 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: overall HEALTH_OK 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:05.003 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:31:05.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: pgmap v12: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.4 MiB/s wr, 561 op/s 2024-09-18T07:31:05.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata", "id": "smithi083"} : dispatch 2024-09-18T07:31:05.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mon metadata", "id": "smithi096"} : dispatch 2024-09-18T07:31:05.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: mon.smithi096 calling monitor election 2024-09-18T07:31:05.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: mon.smithi083 calling monitor election 2024-09-18T07:31:05.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: mon.smithi083 is new leader, mons smithi083,smithi096 in quorum (ranks 0,1) 2024-09-18T07:31:05.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: monmap epoch 3 2024-09-18T07:31:05.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: fsid 126f471a-758e-11ef-bceb-c7b262605968 2024-09-18T07:31:05.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: last_changed 2024-09-18T07:31:03.917446+0000 2024-09-18T07:31:05.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: created 2024-09-18T07:17:52.460906+0000 2024-09-18T07:31:05.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: min_mon_release 19 (squid) 2024-09-18T07:31:05.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: election_strategy: 1 2024-09-18T07:31:05.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: 0: [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon.smithi083 2024-09-18T07:31:05.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: 1: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.smithi096 2024-09-18T07:31:05.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:31:05.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: osdmap e73: 8 total, 8 up, 8 in 2024-09-18T07:31:05.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: mgrmap e42: smithi083.vchvnu(active, since 18s), standbys: smithi096.caifnv 2024-09-18T07:31:05.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: overall HEALTH_OK 2024-09-18T07:31:05.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:05.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:05.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:31:05.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 129062 10.46 MB/sec execute 194 sec latency 58.216 ms 2024-09-18T07:31:06.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 129339 10.44 MB/sec execute 195 sec latency 51.299 ms 2024-09-18T07:31:07.091 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:06 smithi083 ceph-mon[63709]: pgmap v13: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 386 op/s 2024-09-18T07:31:07.266 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:06 smithi096 ceph-mon[49483]: pgmap v13: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 386 op/s 2024-09-18T07:31:07.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 129524 10.41 MB/sec execute 196 sec latency 50.685 ms 2024-09-18T07:31:08.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 129803 10.41 MB/sec execute 197 sec latency 74.368 ms 2024-09-18T07:31:08.859 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:08 smithi083 ceph-mon[63709]: pgmap v14: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 643 op/s 2024-09-18T07:31:08.859 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:08.860 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:08.860 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:08.860 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:09.063 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:08 smithi096 ceph-mon[49483]: pgmap v14: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 643 op/s 2024-09-18T07:31:09.063 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:09.063 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:09.064 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:09.064 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:09.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 130229 10.38 MB/sec execute 198 sec latency 75.285 ms 2024-09-18T07:31:10.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 130636 10.40 MB/sec execute 199 sec latency 30.601 ms 2024-09-18T07:31:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:10 smithi096 ceph-mon[49483]: pgmap v15: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-09-18T07:31:10.895 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:10 smithi083 ceph-mon[63709]: pgmap v15: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-09-18T07:31:11.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 131078 10.42 MB/sec execute 200 sec latency 34.812 ms 2024-09-18T07:31:12.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 131521 10.41 MB/sec execute 201 sec latency 42.314 ms 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: pgmap v16: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: Reconfiguring mon.smithi083 (monmap changed)... 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-18T07:31:12.924 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-18T07:31:12.925 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:12.925 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:12 smithi083 ceph-mon[63709]: Reconfiguring daemon mon.smithi083 on smithi083 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: pgmap v16: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: Reconfiguring mon.smithi083 (monmap changed)... 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-18T07:31:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:13.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:12 smithi096 ceph-mon[49483]: Reconfiguring daemon mon.smithi083 on smithi083 2024-09-18T07:31:13.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 131855 10.42 MB/sec execute 202 sec latency 43.458 ms 2024-09-18T07:31:14.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 132243 10.43 MB/sec execute 203 sec latency 43.299 ms 2024-09-18T07:31:14.972 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:14 smithi083 ceph-mon[63709]: pgmap v17: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 651 op/s 2024-09-18T07:31:14.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:14.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:14.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi083.vchvnu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-18T07:31:14.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mgr services"} : dispatch 2024-09-18T07:31:14.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:15.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:14 smithi096 ceph-mon[49483]: pgmap v17: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 651 op/s 2024-09-18T07:31:15.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:15.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:15.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi083.vchvnu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-18T07:31:15.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mgr services"} : dispatch 2024-09-18T07:31:15.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:15.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 132694 10.42 MB/sec execute 204 sec latency 32.381 ms 2024-09-18T07:31:15.988 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:15 smithi083 ceph-mon[63709]: Reconfiguring mgr.smithi083.vchvnu (monmap changed)... 2024-09-18T07:31:15.989 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:15 smithi083 ceph-mon[63709]: Reconfiguring daemon mgr.smithi083.vchvnu on smithi083 2024-09-18T07:31:16.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:15 smithi096 ceph-mon[49483]: Reconfiguring mgr.smithi083.vchvnu (monmap changed)... 2024-09-18T07:31:16.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:15 smithi096 ceph-mon[49483]: Reconfiguring daemon mgr.smithi083.vchvnu on smithi083 2024-09-18T07:31:16.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 133008 10.42 MB/sec execute 205 sec latency 43.561 ms 2024-09-18T07:31:17.024 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:16 smithi083 ceph-mon[63709]: pgmap v18: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 477 op/s 2024-09-18T07:31:17.025 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:31:17.025 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:17.025 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:17.025 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-18T07:31:17.025 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:17.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:16 smithi096 ceph-mon[49483]: pgmap v18: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 477 op/s 2024-09-18T07:31:17.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:31:17.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:17.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:17.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-18T07:31:17.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:17.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 133444 10.43 MB/sec execute 206 sec latency 27.989 ms 2024-09-18T07:31:17.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:17 smithi083 ceph-mon[63709]: Reconfiguring crash.smithi083 (monmap changed)... 2024-09-18T07:31:17.944 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:17 smithi083 ceph-mon[63709]: Reconfiguring daemon crash.smithi083 on smithi083 2024-09-18T07:31:18.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:17 smithi096 ceph-mon[49483]: Reconfiguring crash.smithi083 (monmap changed)... 2024-09-18T07:31:18.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:17 smithi096 ceph-mon[49483]: Reconfiguring daemon crash.smithi083 on smithi083 2024-09-18T07:31:18.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 133969 10.41 MB/sec execute 207 sec latency 36.005 ms 2024-09-18T07:31:18.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:18 smithi083 ceph-mon[63709]: pgmap v19: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 711 op/s 2024-09-18T07:31:18.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:18 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:18.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:18 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:18.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:18 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-18T07:31:18.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:18 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:19.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:18 smithi096 ceph-mon[49483]: pgmap v19: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 711 op/s 2024-09-18T07:31:19.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:18 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:19.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:18 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:19.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:18 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-18T07:31:19.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:18 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:19.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 134539 10.46 MB/sec execute 208 sec latency 32.045 ms 2024-09-18T07:31:19.977 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:19 smithi083 ceph-mon[63709]: Reconfiguring osd.0 (monmap changed)... 2024-09-18T07:31:19.977 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:19 smithi083 ceph-mon[63709]: Reconfiguring daemon osd.0 on smithi083 2024-09-18T07:31:20.097 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:31:20.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:19 smithi096 ceph-mon[49483]: Reconfiguring osd.0 (monmap changed)... 2024-09-18T07:31:20.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:19 smithi096 ceph-mon[49483]: Reconfiguring daemon osd.0 on smithi083 2024-09-18T07:31:20.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 135313 10.51 MB/sec execute 209 sec latency 25.442 ms 2024-09-18T07:31:20.991 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:20 smithi083 ceph-mon[63709]: pgmap v20: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-18T07:31:21.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:20 smithi096 ceph-mon[49483]: pgmap v20: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-18T07:31:21.175 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:31:21.175 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (2m) 26s ago 12m 19.8M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:31:21.175 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (11m) 26s ago 11m 6413k - 17.2.0 e1d6a67b021e d745b2fecb4e 2024-09-18T07:31:21.175 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (11m) 13s ago 11m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:31:21.175 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (2m) 26s ago 11m 44.6M - dad864ee21e9 06ff04a18f50 2024-09-18T07:31:21.175 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (6m) 26s ago 6m 40.0M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (6m) 13s ago 5m 15.8M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (110s) 26s ago 13m 489M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (60s) 13s ago 11m 452M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (44s) 26s ago 13m 31.3M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (18s) 13s ago 11m 25.0M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (5m) 26s ago 5m 264M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (5m) 13s ago 5m 90.1M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (2m) 26s ago 11m 9977k - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (119s) 13s ago 11m 10.1M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (10m) 26s ago 10m 473M 1936M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (9m) 26s ago 9m 398M 1936M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (9m) 26s ago 9m 290M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9m) 26s ago 9m 421M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:31:21.176 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (8m) 13s ago 8m 471M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:31:21.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (8m) 13s ago 8m 498M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:31:21.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (7m) 13s ago 7m 419M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:31:21.177 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (7m) 13s ago 7m 329M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:31:21.177 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (66s) 26s ago 10m 48.3M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:31:21.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 135721 10.52 MB/sec execute 210 sec latency 33.608 ms 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:31:21.922 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:31:21.922 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:31:21.922 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:31:21.922 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:31:21.922 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-18T07:31:21.922 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 4 2024-09-18T07:31:21.922 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:31:21.922 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:31:22.008 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:21 smithi083 ceph-mon[63709]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:22.008 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:21 smithi083 ceph-mon[63709]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:22.008 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:22.009 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:22.009 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-18T07:31:22.009 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:22.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:21 smithi096 ceph-mon[49483]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:22.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:21 smithi096 ceph-mon[49483]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:22.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:22.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:22.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-18T07:31:22.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:22.470 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:31:22.470 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:31:22.470 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:31:22.471 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:31:22.471 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:31:22.471 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:31:22.471 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:31:22.471 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:31:22.471 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "4/23 daemons upgraded", 2024-09-18T07:31:22.471 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mon daemons", 2024-09-18T07:31:22.471 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:31:22.471 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:31:22.528 INFO:teuthology.orchestra.run.smithi083.stdout: 5 136150 10.53 MB/sec execute 211 sec latency 43.393 ms 2024-09-18T07:31:23.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:22 smithi083 ceph-mon[63709]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:23.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:22 smithi083 ceph-mon[63709]: Reconfiguring osd.1 (monmap changed)... 2024-09-18T07:31:23.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:22 smithi083 ceph-mon[63709]: Reconfiguring daemon osd.1 on smithi083 2024-09-18T07:31:23.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:22 smithi083 ceph-mon[63709]: pgmap v21: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-18T07:31:23.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:22 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1260785265' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:31:23.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:22 smithi096 ceph-mon[49483]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:23.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:22 smithi096 ceph-mon[49483]: Reconfiguring osd.1 (monmap changed)... 2024-09-18T07:31:23.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:22 smithi096 ceph-mon[49483]: Reconfiguring daemon osd.1 on smithi083 2024-09-18T07:31:23.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:22 smithi096 ceph-mon[49483]: pgmap v21: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-18T07:31:23.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:22 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1260785265' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:31:23.155 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:31:23.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 136549 10.52 MB/sec execute 212 sec latency 46.885 ms 2024-09-18T07:31:24.078 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:23 smithi083 ceph-mon[63709]: from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:24.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:23 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:24.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:23 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:24.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:23 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-18T07:31:24.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:23 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:24.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:23 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/512998924' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:31:24.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:23 smithi096 ceph-mon[49483]: from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:23 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:23 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:23 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-18T07:31:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:23 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:23 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/512998924' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:31:24.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 136966 10.53 MB/sec execute 213 sec latency 91.070 ms 2024-09-18T07:31:25.099 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:24 smithi083 ceph-mon[63709]: Reconfiguring osd.2 (monmap changed)... 2024-09-18T07:31:25.099 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:24 smithi083 ceph-mon[63709]: Reconfiguring daemon osd.2 on smithi083 2024-09-18T07:31:25.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:24 smithi083 ceph-mon[63709]: pgmap v22: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 752 op/s 2024-09-18T07:31:25.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:24 smithi096 ceph-mon[49483]: Reconfiguring osd.2 (monmap changed)... 2024-09-18T07:31:25.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:24 smithi096 ceph-mon[49483]: Reconfiguring daemon osd.2 on smithi083 2024-09-18T07:31:25.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:24 smithi096 ceph-mon[49483]: pgmap v22: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 752 op/s 2024-09-18T07:31:25.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 137411 10.52 MB/sec execute 214 sec latency 51.155 ms 2024-09-18T07:31:26.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 137714 10.51 MB/sec execute 215 sec latency 73.387 ms 2024-09-18T07:31:26.930 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:26 smithi083 ceph-mon[63709]: pgmap v23: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.8 MiB/s wr, 552 op/s 2024-09-18T07:31:26.930 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:26.930 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:26.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:26 smithi083 ceph-mon[63709]: Reconfiguring osd.3 (monmap changed)... 2024-09-18T07:31:26.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-18T07:31:26.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:26.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:26 smithi083 ceph-mon[63709]: Reconfiguring daemon osd.3 on smithi083 2024-09-18T07:31:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:26 smithi096 ceph-mon[49483]: pgmap v23: 97 pgs: 97 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.8 MiB/s wr, 552 op/s 2024-09-18T07:31:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:26 smithi096 ceph-mon[49483]: Reconfiguring osd.3 (monmap changed)... 2024-09-18T07:31:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-18T07:31:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:26 smithi096 ceph-mon[49483]: Reconfiguring daemon osd.3 on smithi083 2024-09-18T07:31:27.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 137998 10.50 MB/sec execute 216 sec latency 55.398 ms 2024-09-18T07:31:28.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 138353 10.50 MB/sec execute 217 sec latency 73.470 ms 2024-09-18T07:31:28.729 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:28 smithi083 ceph-mon[63709]: pgmap v24: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 798 op/s 2024-09-18T07:31:28.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:28.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:28.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:28 smithi083 ceph-mon[63709]: Reconfiguring mds.foofs.smithi083.kjbbry (monmap changed)... 2024-09-18T07:31:28.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.kjbbry", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-18T07:31:28.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:28.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:28 smithi083 ceph-mon[63709]: Reconfiguring daemon mds.foofs.smithi083.kjbbry on smithi083 2024-09-18T07:31:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:28 smithi096 ceph-mon[49483]: pgmap v24: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 798 op/s 2024-09-18T07:31:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:28 smithi096 ceph-mon[49483]: Reconfiguring mds.foofs.smithi083.kjbbry (monmap changed)... 2024-09-18T07:31:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.kjbbry", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-18T07:31:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:28 smithi096 ceph-mon[49483]: Reconfiguring daemon mds.foofs.smithi083.kjbbry on smithi083 2024-09-18T07:31:29.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 138719 10.49 MB/sec execute 218 sec latency 77.608 ms 2024-09-18T07:31:30.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 139092 10.50 MB/sec execute 219 sec latency 43.359 ms 2024-09-18T07:31:30.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:30 smithi083 ceph-mon[63709]: pgmap v25: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 563 op/s 2024-09-18T07:31:30.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:30 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:30.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:30 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:30.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:30 smithi083 ceph-mon[63709]: Reconfiguring crash.smithi096 (monmap changed)... 2024-09-18T07:31:30.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:30 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi096", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-18T07:31:30.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:30 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:30.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:30 smithi083 ceph-mon[63709]: Reconfiguring daemon crash.smithi096 on smithi096 2024-09-18T07:31:30.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:30 smithi096 ceph-mon[49483]: pgmap v25: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 563 op/s 2024-09-18T07:31:30.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:30 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:30 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:30 smithi096 ceph-mon[49483]: Reconfiguring crash.smithi096 (monmap changed)... 2024-09-18T07:31:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:30 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi096", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-18T07:31:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:30 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:30 smithi096 ceph-mon[49483]: Reconfiguring daemon crash.smithi096 on smithi096 2024-09-18T07:31:31.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 139441 10.51 MB/sec execute 220 sec latency 47.560 ms 2024-09-18T07:31:31.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:31:31.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:31.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:31.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-18T07:31:31.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mgr services"} : dispatch 2024-09-18T07:31:31.785 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:31.818 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:31:31.819 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:31.819 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:31.819 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi096.caifnv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-18T07:31:31.819 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mgr services"} : dispatch 2024-09-18T07:31:31.819 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:32.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 139820 10.49 MB/sec execute 221 sec latency 69.499 ms 2024-09-18T07:31:32.830 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:32 smithi083 ceph-mon[63709]: Reconfiguring mgr.smithi096.caifnv (monmap changed)... 2024-09-18T07:31:32.831 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:32 smithi083 ceph-mon[63709]: Reconfiguring daemon mgr.smithi096.caifnv on smithi096 2024-09-18T07:31:32.831 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:32 smithi083 ceph-mon[63709]: pgmap v26: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.3 MiB/s wr, 591 op/s 2024-09-18T07:31:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:32 smithi096 ceph-mon[49483]: Reconfiguring mgr.smithi096.caifnv (monmap changed)... 2024-09-18T07:31:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:32 smithi096 ceph-mon[49483]: Reconfiguring daemon mgr.smithi096.caifnv on smithi096 2024-09-18T07:31:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:32 smithi096 ceph-mon[49483]: pgmap v26: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.3 MiB/s wr, 591 op/s 2024-09-18T07:31:33.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 140097 10.48 MB/sec execute 222 sec latency 43.492 ms 2024-09-18T07:31:33.846 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:33 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:33.846 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:33 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:33.846 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:33 smithi083 ceph-mon[63709]: Reconfiguring mon.smithi096 (monmap changed)... 2024-09-18T07:31:33.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:33 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-18T07:31:33.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:33 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-18T07:31:33.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:33 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:33.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:33 smithi083 ceph-mon[63709]: Reconfiguring daemon mon.smithi096 on smithi096 2024-09-18T07:31:33.976 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:33 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:33.976 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:33 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:33.976 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:33 smithi096 ceph-mon[49483]: Reconfiguring mon.smithi096 (monmap changed)... 2024-09-18T07:31:33.976 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:33 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-18T07:31:33.976 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:33 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-18T07:31:33.976 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:33 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:33.976 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:33 smithi096 ceph-mon[49483]: Reconfiguring daemon mon.smithi096 on smithi096 2024-09-18T07:31:34.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 140424 10.47 MB/sec execute 223 sec latency 67.157 ms 2024-09-18T07:31:34.640 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:34 smithi096 ceph-mon[49483]: pgmap v27: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 753 op/s 2024-09-18T07:31:34.640 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:34 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:34.688 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:34 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:34.688 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:34 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-18T07:31:34.688 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:34 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:34.880 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:34 smithi083 ceph-mon[63709]: pgmap v27: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 753 op/s 2024-09-18T07:31:34.880 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:34 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:34.881 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:34 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:34.881 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:34 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-18T07:31:34.881 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:34 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:35.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 140793 10.46 MB/sec execute 224 sec latency 43.959 ms 2024-09-18T07:31:35.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:35 smithi096 ceph-mon[49483]: Reconfiguring osd.4 (monmap changed)... 2024-09-18T07:31:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:35 smithi096 ceph-mon[49483]: Reconfiguring daemon osd.4 on smithi096 2024-09-18T07:31:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-18T07:31:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:35.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:35 smithi083 ceph-mon[63709]: Reconfiguring osd.4 (monmap changed)... 2024-09-18T07:31:35.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:35 smithi083 ceph-mon[63709]: Reconfiguring daemon osd.4 on smithi096 2024-09-18T07:31:35.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:35.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:35.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-18T07:31:35.900 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:36.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 141055 10.44 MB/sec execute 225 sec latency 78.014 ms 2024-09-18T07:31:36.660 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:36 smithi083 ceph-mon[63709]: pgmap v28: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 449 op/s 2024-09-18T07:31:36.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:36 smithi083 ceph-mon[63709]: Reconfiguring osd.5 (monmap changed)... 2024-09-18T07:31:36.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:36 smithi083 ceph-mon[63709]: Reconfiguring daemon osd.5 on smithi096 2024-09-18T07:31:36.956 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:36 smithi096 ceph-mon[49483]: pgmap v28: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 449 op/s 2024-09-18T07:31:36.956 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:36 smithi096 ceph-mon[49483]: Reconfiguring osd.5 (monmap changed)... 2024-09-18T07:31:36.956 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:36 smithi096 ceph-mon[49483]: Reconfiguring daemon osd.5 on smithi096 2024-09-18T07:31:37.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 141330 10.43 MB/sec execute 226 sec latency 84.550 ms 2024-09-18T07:31:38.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:38.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:38.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:38 smithi083 ceph-mon[63709]: Reconfiguring osd.6 (monmap changed)... 2024-09-18T07:31:38.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-18T07:31:38.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:38.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:38 smithi083 ceph-mon[63709]: Reconfiguring daemon osd.6 on smithi096 2024-09-18T07:31:38.205 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:38 smithi083 ceph-mon[63709]: pgmap v29: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 655 op/s 2024-09-18T07:31:38.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:38.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:38.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:38 smithi096 ceph-mon[49483]: Reconfiguring osd.6 (monmap changed)... 2024-09-18T07:31:38.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-18T07:31:38.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:38.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:38 smithi096 ceph-mon[49483]: Reconfiguring daemon osd.6 on smithi096 2024-09-18T07:31:38.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:38 smithi096 ceph-mon[49483]: pgmap v29: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 655 op/s 2024-09-18T07:31:38.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 141665 10.43 MB/sec execute 227 sec latency 49.093 ms 2024-09-18T07:31:39.529 INFO:teuthology.orchestra.run.smithi083.stdout: 5 141988 10.41 MB/sec execute 228 sec latency 73.334 ms 2024-09-18T07:31:39.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:39 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:39.746 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:39 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:39.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:39 smithi083 ceph-mon[63709]: Reconfiguring osd.7 (monmap changed)... 2024-09-18T07:31:39.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:39 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-18T07:31:39.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:39 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:39.747 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:39 smithi083 ceph-mon[63709]: Reconfiguring daemon osd.7 on smithi096 2024-09-18T07:31:39.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:39 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:39 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:39 smithi096 ceph-mon[49483]: Reconfiguring osd.7 (monmap changed)... 2024-09-18T07:31:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:39 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-18T07:31:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:39 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:39 smithi096 ceph-mon[49483]: Reconfiguring daemon osd.7 on smithi096 2024-09-18T07:31:40.534 INFO:teuthology.orchestra.run.smithi083.stdout: 5 142280 10.40 MB/sec execute 229 sec latency 52.487 ms 2024-09-18T07:31:40.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:40 smithi083 ceph-mon[63709]: pgmap v30: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 409 op/s 2024-09-18T07:31:40.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:40 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:40.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:40 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:40.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:40 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi096.bzilgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-18T07:31:40.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:40 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:40.821 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:40 smithi096 ceph-mon[49483]: pgmap v30: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 409 op/s 2024-09-18T07:31:40.821 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:40 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:40.821 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:40 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:40.821 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:40 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi096.bzilgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-18T07:31:40.821 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:40 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:41.534 INFO:teuthology.orchestra.run.smithi083.stdout: 5 142815 10.44 MB/sec execute 230 sec latency 92.834 ms 2024-09-18T07:31:41.696 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:41 smithi096 ceph-mon[49483]: Reconfiguring mds.foofs.smithi096.bzilgv (monmap changed)... 2024-09-18T07:31:41.697 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:41 smithi096 ceph-mon[49483]: Reconfiguring daemon mds.foofs.smithi096.bzilgv on smithi096 2024-09-18T07:31:41.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:41 smithi083 ceph-mon[63709]: Reconfiguring mds.foofs.smithi096.bzilgv (monmap changed)... 2024-09-18T07:31:41.764 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:41 smithi083 ceph-mon[63709]: Reconfiguring daemon mds.foofs.smithi096.bzilgv on smithi096 2024-09-18T07:31:42.534 INFO:teuthology.orchestra.run.smithi083.stdout: 5 143334 10.46 MB/sec execute 231 sec latency 60.594 ms 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: pgmap v31: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 573 op/s 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi083"} : dispatch 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi083"}]': finished 2024-09-18T07:31:42.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi096"} : dispatch 2024-09-18T07:31:42.770 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi096"}]': finished 2024-09-18T07:31:43.084 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: pgmap v31: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 573 op/s 2024-09-18T07:31:43.084 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:43.084 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:43.084 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:31:43.084 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:31:43.084 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:31:43.084 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:43.085 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi083"} : dispatch 2024-09-18T07:31:43.085 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi083"}]': finished 2024-09-18T07:31:43.085 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi096"} : dispatch 2024-09-18T07:31:43.085 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi096"}]': finished 2024-09-18T07:31:43.534 INFO:teuthology.orchestra.run.smithi083.stdout: 5 143782 10.45 MB/sec execute 232 sec latency 46.021 ms 2024-09-18T07:31:43.783 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:43 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all mon 2024-09-18T07:31:44.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:43 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all mon 2024-09-18T07:31:44.534 INFO:teuthology.orchestra.run.smithi083.stdout: 5 144259 10.46 MB/sec execute 233 sec latency 32.997 ms 2024-09-18T07:31:44.793 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:44 smithi083 ceph-mon[63709]: pgmap v32: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 594 op/s 2024-09-18T07:31:44.794 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:44 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:44.794 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:44 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-18T07:31:44.794 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:44 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:45.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:44 smithi096 ceph-mon[49483]: pgmap v32: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 594 op/s 2024-09-18T07:31:45.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:44 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:45.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:44 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi083", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-18T07:31:45.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:44 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:45.534 INFO:teuthology.orchestra.run.smithi083.stdout: 5 144742 10.47 MB/sec execute 234 sec latency 38.530 ms 2024-09-18T07:31:45.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:45 smithi083 ceph-mon[63709]: Upgrade: Updating crash.smithi083 (1/2) 2024-09-18T07:31:45.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:45 smithi083 ceph-mon[63709]: Deploying daemon crash.smithi083 on smithi083 2024-09-18T07:31:46.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:45 smithi096 ceph-mon[49483]: Upgrade: Updating crash.smithi083 (1/2) 2024-09-18T07:31:46.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:45 smithi096 ceph-mon[49483]: Deploying daemon crash.smithi083 on smithi083 2024-09-18T07:31:46.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 145271 10.48 MB/sec execute 235 sec latency 39.719 ms 2024-09-18T07:31:46.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:46 smithi083 ceph-mon[63709]: pgmap v33: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 417 op/s 2024-09-18T07:31:46.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:31:47.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:46 smithi096 ceph-mon[49483]: pgmap v33: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 417 op/s 2024-09-18T07:31:47.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:31:47.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 145717 10.49 MB/sec execute 236 sec latency 37.102 ms 2024-09-18T07:31:48.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 146104 10.50 MB/sec execute 237 sec latency 47.153 ms 2024-09-18T07:31:48.842 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:48 smithi083 ceph-mon[63709]: pgmap v34: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 713 op/s 2024-09-18T07:31:49.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:48 smithi096 ceph-mon[49483]: pgmap v34: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 713 op/s 2024-09-18T07:31:49.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 146479 10.51 MB/sec execute 238 sec latency 57.358 ms 2024-09-18T07:31:50.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 146912 10.51 MB/sec execute 239 sec latency 60.634 ms 2024-09-18T07:31:50.861 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:50 smithi083 ceph-mon[63709]: pgmap v35: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 508 op/s 2024-09-18T07:31:50.862 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:50.862 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:51.009 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:50 smithi096 ceph-mon[49483]: pgmap v35: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 508 op/s 2024-09-18T07:31:51.009 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:51.009 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:51.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 147445 10.51 MB/sec execute 240 sec latency 40.410 ms 2024-09-18T07:31:52.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 147769 10.50 MB/sec execute 241 sec latency 63.623 ms 2024-09-18T07:31:52.956 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:52 smithi083 ceph-mon[63709]: pgmap v36: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 698 op/s 2024-09-18T07:31:52.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:52 smithi083 ceph-mon[63709]: Upgrade: Updating crash.smithi096 (2/2) 2024-09-18T07:31:52.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:52.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi096", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-18T07:31:52.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:52.957 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:52 smithi083 ceph-mon[63709]: Deploying daemon crash.smithi096 on smithi096 2024-09-18T07:31:53.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:52 smithi096 ceph-mon[49483]: pgmap v36: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 698 op/s 2024-09-18T07:31:53.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:52 smithi096 ceph-mon[49483]: Upgrade: Updating crash.smithi096 (2/2) 2024-09-18T07:31:53.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:53.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi096", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-18T07:31:53.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:31:53.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:52 smithi096 ceph-mon[49483]: Deploying daemon crash.smithi096 on smithi096 2024-09-18T07:31:53.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 148292 10.52 MB/sec execute 242 sec latency 32.494 ms 2024-09-18T07:31:53.777 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:31:54.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 148778 10.52 MB/sec execute 243 sec latency 34.755 ms 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (3m) 60s ago 12m 19.8M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 starting - - - - 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (12m) 47s ago 11m 6409k - 17.2.0 e1d6a67b021e 33c4d8abac06 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (3m) 60s ago 12m 44.6M - dad864ee21e9 06ff04a18f50 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (6m) 60s ago 6m 40.0M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (6m) 47s ago 6m 15.8M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (2m) 60s ago 13m 489M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (94s) 47s ago 11m 452M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (78s) 60s ago 13m 31.3M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (52s) 47s ago 11m 25.0M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:31:54.967 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (6m) 60s ago 6m 264M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (6m) 47s ago 6m 90.1M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (3m) 60s ago 12m 9977k - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (2m) 47s ago 11m 10.1M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11m) 60s ago 11m 473M 1936M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (10m) 60s ago 10m 398M 1936M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10m) 60s ago 10m 290M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (9m) 60s ago 9m 421M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (9m) 47s ago 9m 471M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (8m) 47s ago 8m 498M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (8m) 47s ago 8m 419M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (8m) 47s ago 8m 329M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:31:54.968 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (100s) 60s ago 11m 48.3M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:31:54.992 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:54 smithi083 ceph-mon[63709]: pgmap v37: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-18T07:31:54.993 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:54 smithi083 ceph-mon[63709]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:55.109 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:54 smithi096 ceph-mon[49483]: pgmap v37: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-09-18T07:31:55.109 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:54 smithi096 ceph-mon[49483]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:55.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 149226 10.53 MB/sec execute 244 sec latency 41.003 ms 2024-09-18T07:31:55.789 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:31:55.789 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:31:55.789 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:31:55.789 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:31:55.789 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 4 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:31:55.790 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:31:56.006 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:55 smithi083 ceph-mon[63709]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:56.007 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:55 smithi083 ceph-mon[63709]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:56.007 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:55 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:56.007 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:55 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:56.007 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:55 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:31:56.007 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:55 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1779087738' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:31:56.116 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:55 smithi096 ceph-mon[49483]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:56.116 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:55 smithi096 ceph-mon[49483]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:56.116 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:55 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:56.116 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:55 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:31:56.116 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:55 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:31:56.117 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:55 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1779087738' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:31:56.373 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "4/23 daemons upgraded", 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading crash daemons", 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:31:56.374 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:31:56.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 149540 10.53 MB/sec execute 245 sec latency 42.701 ms 2024-09-18T07:31:56.996 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:31:57.033 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:56 smithi083 ceph-mon[63709]: pgmap v38: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.6 MiB/s wr, 563 op/s 2024-09-18T07:31:57.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:56 smithi096 ceph-mon[49483]: pgmap v38: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.6 MiB/s wr, 563 op/s 2024-09-18T07:31:57.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 149900 10.54 MB/sec execute 246 sec latency 63.260 ms 2024-09-18T07:31:58.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:57 smithi083 ceph-mon[63709]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:58.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:57 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/957410635' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:31:58.119 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:57 smithi096 ceph-mon[49483]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:31:58.120 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:57 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/957410635' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:31:58.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 150217 10.54 MB/sec execute 247 sec latency 51.505 ms 2024-09-18T07:31:59.041 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:31:58 smithi096 ceph-mon[49483]: pgmap v39: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 815 op/s 2024-09-18T07:31:59.070 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:31:58 smithi083 ceph-mon[63709]: pgmap v39: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 815 op/s 2024-09-18T07:31:59.537 INFO:teuthology.orchestra.run.smithi083.stdout: 5 150485 10.52 MB/sec execute 248 sec latency 73.553 ms 2024-09-18T07:32:00.341 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:00 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:00.342 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:00 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:00.342 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:00 smithi083 ceph-mon[63709]: pgmap v40: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.5 MiB/s wr, 520 op/s 2024-09-18T07:32:00.342 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:00 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:00.342 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:00 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:00.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 150714 10.49 MB/sec execute 249 sec latency 99.831 ms 2024-09-18T07:32:00.596 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:00 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:00.597 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:00 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:00.597 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:00 smithi096 ceph-mon[49483]: pgmap v40: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.5 MiB/s wr, 520 op/s 2024-09-18T07:32:00.597 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:00 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:00.597 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:00 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:01.395 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:32:01.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 150959 10.47 MB/sec execute 250 sec latency 72.958 ms 2024-09-18T07:32:01.553 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:32:02.433 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:02 smithi083 ceph-mon[63709]: pgmap v41: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 690 op/s 2024-09-18T07:32:02.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 151302 10.46 MB/sec execute 251 sec latency 63.021 ms 2024-09-18T07:32:02.569 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:02 smithi096 ceph-mon[49483]: pgmap v41: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 690 op/s 2024-09-18T07:32:03.535 INFO:teuthology.orchestra.run.smithi083.stdout: 5 151557 10.45 MB/sec execute 252 sec latency 68.762 ms 2024-09-18T07:32:04.475 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:04.475 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:04.476 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:04 smithi083 ceph-mon[63709]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 512 op/s 2024-09-18T07:32:04.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 151802 10.44 MB/sec execute 253 sec latency 66.745 ms 2024-09-18T07:32:04.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:04 smithi096 ceph-mon[49483]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 512 op/s 2024-09-18T07:32:05.499 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:05 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:05.500 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:05 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:05.500 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:05 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:05.500 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:05 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:05.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 152142 10.44 MB/sec execute 254 sec latency 93.052 ms 2024-09-18T07:32:05.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:05 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:05.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:05 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:05.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:05 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:05.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:05 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:06.524 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:06 smithi083 ceph-mon[63709]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-18T07:32:06.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 152665 10.44 MB/sec execute 255 sec latency 45.664 ms 2024-09-18T07:32:06.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:06 smithi096 ceph-mon[49483]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-18T07:32:07.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 153321 10.49 MB/sec execute 256 sec latency 41.748 ms 2024-09-18T07:32:08.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 153856 10.51 MB/sec execute 257 sec latency 55.346 ms 2024-09-18T07:32:08.702 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:08 smithi083 ceph-mon[63709]: pgmap v44: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.2 MiB/s wr, 629 op/s 2024-09-18T07:32:08.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:08 smithi096 ceph-mon[49483]: pgmap v44: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.2 MiB/s wr, 629 op/s 2024-09-18T07:32:09.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 154347 10.50 MB/sec execute 258 sec latency 39.369 ms 2024-09-18T07:32:10.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 154841 10.52 MB/sec execute 259 sec latency 71.056 ms 2024-09-18T07:32:10.800 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:10 smithi083 ceph-mon[63709]: pgmap v45: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 376 op/s 2024-09-18T07:32:10.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:10 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:10.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:10 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:10.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:10 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:32:10.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:10 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:32:10.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:10 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:10.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:10 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:10 smithi096 ceph-mon[49483]: pgmap v45: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 376 op/s 2024-09-18T07:32:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:10 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:10 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:10 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:32:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:10 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:32:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:10 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:10 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:11.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 155364 10.52 MB/sec execute 260 sec latency 34.603 ms 2024-09-18T07:32:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:32:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all crash 2024-09-18T07:32:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi083"} : dispatch 2024-09-18T07:32:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi083"}]': finished 2024-09-18T07:32:11.806 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi096"} : dispatch 2024-09-18T07:32:11.807 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi096"}]': finished 2024-09-18T07:32:11.807 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-18T07:32:11.807 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-18T07:32:11.807 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:11 smithi083 ceph-mon[63709]: Upgrade: osd.0 is safe to restart 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all crash 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi083"} : dispatch 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi083"}]': finished 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi096"} : dispatch 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi096"}]': finished 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-18T07:32:11.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-18T07:32:11.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:11 smithi096 ceph-mon[49483]: Upgrade: osd.0 is safe to restart 2024-09-18T07:32:12.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 155740 10.51 MB/sec execute 261 sec latency 59.375 ms 2024-09-18T07:32:13.109 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:12 smithi083 ceph-mon[63709]: pgmap v46: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 554 op/s 2024-09-18T07:32:13.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:12 smithi096 ceph-mon[49483]: pgmap v46: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 554 op/s 2024-09-18T07:32:13.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 156039 10.50 MB/sec execute 262 sec latency 71.532 ms 2024-09-18T07:32:14.361 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:14 smithi083 ceph-mon[63709]: Upgrade: Updating osd.0 2024-09-18T07:32:14.362 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:14.362 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-18T07:32:14.362 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:32:14.362 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:14 smithi083 ceph-mon[63709]: Deploying daemon osd.0 on smithi083 2024-09-18T07:32:14.362 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:14 smithi083 ceph-mon[63709]: pgmap v47: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.6 MiB/s wr, 539 op/s 2024-09-18T07:32:14.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 156458 10.51 MB/sec execute 263 sec latency 34.093 ms 2024-09-18T07:32:14.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:14 smithi096 ceph-mon[49483]: Upgrade: Updating osd.0 2024-09-18T07:32:14.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:14.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-18T07:32:14.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:32:14.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:14 smithi096 ceph-mon[49483]: Deploying daemon osd.0 on smithi083 2024-09-18T07:32:14.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:14 smithi096 ceph-mon[49483]: pgmap v47: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.6 MiB/s wr, 539 op/s 2024-09-18T07:32:15.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 156734 10.50 MB/sec execute 264 sec latency 61.613 ms 2024-09-18T07:32:16.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 157036 10.52 MB/sec execute 265 sec latency 54.787 ms 2024-09-18T07:32:16.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:16 smithi083 ceph-mon[63709]: pgmap v48: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.3 MiB/s wr, 526 op/s 2024-09-18T07:32:16.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:32:16.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:16 smithi096 ceph-mon[49483]: pgmap v48: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.3 MiB/s wr, 526 op/s 2024-09-18T07:32:16.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:32:17.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 157314 10.51 MB/sec execute 266 sec latency 51.817 ms 2024-09-18T07:32:17.938 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:17 smithi083 systemd[1]: Stopping Ceph osd.0 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:32:18.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 157608 10.49 MB/sec execute 267 sec latency 55.419 ms 2024-09-18T07:32:18.712 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:18 smithi083 ceph-mon[63709]: pgmap v49: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 725 op/s 2024-09-18T07:32:18.712 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:18 smithi083 ceph-mon[63709]: mgrmap e43: smithi083.vchvnu(active, since 92s), standbys: smithi096.caifnv 2024-09-18T07:32:18.712 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:18 smithi083 ceph-mon[63709]: osd.0 marked itself down and dead 2024-09-18T07:32:18.713 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:18 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[34168]: 2024-09-18T07:32:18.497+0000 7fc55ee97700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:32:18.713 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:18 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[34168]: 2024-09-18T07:32:18.497+0000 7fc55ee97700 -1 osd.0 73 *** Got signal Terminated *** 2024-09-18T07:32:18.713 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:18 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[34168]: 2024-09-18T07:32:18.497+0000 7fc55ee97700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:32:18.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:18 smithi096 ceph-mon[49483]: pgmap v49: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 725 op/s 2024-09-18T07:32:18.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:18 smithi096 ceph-mon[49483]: mgrmap e43: smithi083.vchvnu(active, since 92s), standbys: smithi096.caifnv 2024-09-18T07:32:18.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:18 smithi096 ceph-mon[49483]: osd.0 marked itself down and dead 2024-09-18T07:32:19.489 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:19 smithi083 podman[68003]: 2024-09-18 07:32:19.353266969 +0000 UTC m=+1.310965453 container died 1a9adbb1aa193a5e922dfe36d85bc4e13c0da0aef39371c376040623784df8e1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, ceph=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.openshift.expose-services=, release=754) 2024-09-18T07:32:19.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 157811 10.45 MB/sec execute 268 sec latency 890.553 ms 2024-09-18T07:32:19.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:19 smithi083 ceph-mon[63709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:32:19.744 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:19 smithi083 ceph-mon[63709]: osdmap e74: 8 total, 7 up, 8 in 2024-09-18T07:32:19.744 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:19 smithi083 ceph-mon[63709]: osdmap e75: 8 total, 7 up, 8 in 2024-09-18T07:32:19.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:19 smithi096 ceph-mon[49483]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:32:19.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:19 smithi096 ceph-mon[49483]: osdmap e74: 8 total, 7 up, 8 in 2024-09-18T07:32:19.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:19 smithi096 ceph-mon[49483]: osdmap e75: 8 total, 7 up, 8 in 2024-09-18T07:32:20.261 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:20 smithi083 podman[68003]: 2024-09-18 07:32:20.179169633 +0000 UTC m=+2.136868119 container cleanup 1a9adbb1aa193a5e922dfe36d85bc4e13c0da0aef39371c376040623784df8e1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0, io.openshift.expose-services=, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, ceph=True, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, maintainer=Guillaume Abrioux , RELEASE=HEAD, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-18T07:32:20.261 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:20 smithi083 bash[68003]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0 2024-09-18T07:32:20.533 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:20 smithi083 podman[68021]: 2024-09-18 07:32:20.491081284 +0000 UTC m=+1.139618546 container remove 1a9adbb1aa193a5e922dfe36d85bc4e13c0da0aef39371c376040623784df8e1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vendor=Red Hat, Inc., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, release=754, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , version=8, GIT_BRANCH=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8) 2024-09-18T07:32:20.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 158097 10.44 MB/sec execute 269 sec latency 967.665 ms 2024-09-18T07:32:20.785 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:20 smithi083 ceph-mon[63709]: pgmap v51: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.8 MiB/s wr, 637 op/s 2024-09-18T07:32:21.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:20 smithi096 ceph-mon[49483]: pgmap v51: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.8 MiB/s wr, 637 op/s 2024-09-18T07:32:21.288 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:21 smithi083 podman[68104]: 2024-09-18 07:32:21.037404131 +0000 UTC m=+0.024819073 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:32:21.289 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:21 smithi083 podman[68104]: 2024-09-18 07:32:21.241698154 +0000 UTC m=+0.229113091 container create e72d2848ee77440f8e31614a5a5b02c84dbd1b47051c73c16d27d723afe4c392 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-18T07:32:21.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 158717 10.48 MB/sec execute 270 sec latency 35.363 ms 2024-09-18T07:32:21.793 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:21 smithi083 ceph-mon[63709]: Health check failed: Degraded data redundancy: 154/1500 objects degraded (10.267%), 22 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:21.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:21 smithi096 ceph-mon[49483]: Health check failed: Degraded data redundancy: 154/1500 objects degraded (10.267%), 22 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:22.045 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:21 smithi083 podman[68104]: 2024-09-18 07:32:21.802735424 +0000 UTC m=+0.790150367 container init e72d2848ee77440f8e31614a5a5b02c84dbd1b47051c73c16d27d723afe4c392 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:32:22.046 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:21 smithi083 podman[68104]: 2024-09-18 07:32:21.812739955 +0000 UTC m=+0.800154891 container start e72d2848ee77440f8e31614a5a5b02c84dbd1b47051c73c16d27d723afe4c392 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-18T07:32:22.046 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:21 smithi083 podman[68104]: 2024-09-18 07:32:21.988358807 +0000 UTC m=+0.975773739 container attach e72d2848ee77440f8e31614a5a5b02c84dbd1b47051c73c16d27d723afe4c392 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:32:22.299 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:22 smithi083 podman[68104]: 2024-09-18 07:32:22.131900386 +0000 UTC m=+1.119315327 container died e72d2848ee77440f8e31614a5a5b02c84dbd1b47051c73c16d27d723afe4c392 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-18T07:32:22.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 159141 10.46 MB/sec execute 271 sec latency 62.566 ms 2024-09-18T07:32:22.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:22 smithi083 ceph-mon[63709]: pgmap v53: 97 pgs: 6 active+undersized, 1 stale+active+clean, 22 active+undersized+degraded, 68 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 607 op/s; 154/1500 objects degraded (10.267%) 2024-09-18T07:32:23.052 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:23 smithi083 podman[68127]: 2024-09-18 07:32:23.014316874 +0000 UTC m=+0.874434351 container remove e72d2848ee77440f8e31614a5a5b02c84dbd1b47051c73c16d27d723afe4c392 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-18T07:32:23.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:22 smithi096 ceph-mon[49483]: pgmap v53: 97 pgs: 6 active+undersized, 1 stale+active+clean, 22 active+undersized+degraded, 68 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 607 op/s; 154/1500 objects degraded (10.267%) 2024-09-18T07:32:23.303 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:23 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-18T07:32:23.303 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:23 smithi083 systemd[1]: Stopped Ceph osd.0 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:32:23.304 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:23 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 41.877s CPU time. 2024-09-18T07:32:23.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 159537 10.45 MB/sec execute 272 sec latency 40.454 ms 2024-09-18T07:32:23.554 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:23 smithi083 systemd[1]: Starting Ceph osd.0 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:32:23.805 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:23 smithi083 podman[68218]: 2024-09-18 07:32:23.568670882 +0000 UTC m=+0.023427693 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:32:24.057 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:23 smithi083 podman[68218]: 2024-09-18 07:32:23.888395433 +0000 UTC m=+0.343152228 container create cb559e77403938a6e9f841088bf109858226025564a89425c9b8a1e0a66ca42d (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-18T07:32:24.543 INFO:teuthology.orchestra.run.smithi083.stdout: 5 160482 10.55 MB/sec execute 273 sec latency 24.158 ms 2024-09-18T07:32:24.559 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:24 smithi083 podman[68218]: 2024-09-18 07:32:24.317696107 +0000 UTC m=+0.772452915 container init cb559e77403938a6e9f841088bf109858226025564a89425c9b8a1e0a66ca42d (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-18T07:32:24.559 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:24 smithi083 podman[68218]: 2024-09-18 07:32:24.324427724 +0000 UTC m=+0.779184526 container start cb559e77403938a6e9f841088bf109858226025564a89425c9b8a1e0a66ca42d (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-18T07:32:24.559 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:24 smithi083 podman[68218]: 2024-09-18 07:32:24.493217296 +0000 UTC m=+0.947974096 container attach cb559e77403938a6e9f841088bf109858226025564a89425c9b8a1e0a66ca42d (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-18T07:32:24.559 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:24 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate[68233]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:32:24.559 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:24 smithi083 bash[68218]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:32:24.559 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:24 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate[68233]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:32:24.559 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:24 smithi083 bash[68218]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:32:24.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:24 smithi083 ceph-mon[63709]: pgmap v54: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 626 op/s; 196/1482 objects degraded (13.225%) 2024-09-18T07:32:25.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:24 smithi096 ceph-mon[49483]: pgmap v54: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 626 op/s; 196/1482 objects degraded (13.225%) 2024-09-18T07:32:25.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 160997 10.57 MB/sec execute 274 sec latency 31.363 ms 2024-09-18T07:32:26.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 161287 10.53 MB/sec execute 275 sec latency 82.176 ms 2024-09-18T07:32:26.873 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:26 smithi083 ceph-mon[63709]: pgmap v55: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.9 MiB/s wr, 327 op/s; 196/1482 objects degraded (13.225%) 2024-09-18T07:32:27.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:26 smithi096 ceph-mon[49483]: pgmap v55: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.9 MiB/s wr, 327 op/s; 196/1482 objects degraded (13.225%) 2024-09-18T07:32:27.486 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:32:27.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 161521 10.51 MB/sec execute 276 sec latency 80.430 ms 2024-09-18T07:32:27.654 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate[68233]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-18T07:32:27.655 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 bash[68218]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-18T07:32:27.655 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate[68233]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-18T07:32:27.655 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 bash[68218]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-18T07:32:27.908 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:27 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 196/1482 objects degraded (13.225%), 29 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:27.909 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate[68233]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-18T07:32:27.909 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 bash[68218]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-18T07:32:27.909 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate[68233]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-18T07:32:27.909 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 bash[68218]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-18T07:32:27.909 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate[68233]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-18T07:32:27.909 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 bash[68218]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-18T07:32:27.909 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate[68233]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-18T07:32:27.909 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 bash[68218]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-18T07:32:28.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:27 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 196/1482 objects degraded (13.225%), 29 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:28.162 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate[68233]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-18T07:32:28.163 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 bash[68218]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-18T07:32:28.163 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:27 smithi083 podman[68218]: 2024-09-18 07:32:27.946494159 +0000 UTC m=+4.401250959 container died cb559e77403938a6e9f841088bf109858226025564a89425c9b8a1e0a66ca42d (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (3m) 23s ago 13m 18.8M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (38s) 23s ago 13m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (33s) 29s ago 12m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (3m) 23s ago 12m 45.9M - dad864ee21e9 06ff04a18f50 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (7m) 23s ago 7m 39.9M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (7m) 29s ago 7m 16.0M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (2m) 23s ago 14m 509M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (2m) 29s ago 12m 453M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (111s) 23s ago 14m 44.2M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (85s) 29s ago 12m 35.8M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (6m) 23s ago 6m 267M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:32:28.419 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (6m) 29s ago 6m 90.1M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (3m) 23s ago 12m 10.0M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (3m) 29s ago 12m 12.2M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (11m) 23s ago 11m 555M 1936M 17.2.0 e1d6a67b021e 1a9adbb1aa19 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11m) 23s ago 11m 469M 1936M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (10m) 23s ago 10m 349M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10m) 23s ago 10m 488M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (9m) 29s ago 9m 507M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (9m) 29s ago 9m 567M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (9m) 29s ago 9m 502M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (8m) 29s ago 8m 388M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:32:28.420 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (2m) 23s ago 12m 48.6M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:32:28.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 161723 10.49 MB/sec execute 277 sec latency 48.922 ms 2024-09-18T07:32:28.930 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:28 smithi083 podman[68448]: 2024-09-18 07:32:28.715812336 +0000 UTC m=+0.762255322 container remove cb559e77403938a6e9f841088bf109858226025564a89425c9b8a1e0a66ca42d (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-18T07:32:28.930 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:28 smithi083 ceph-mon[63709]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:32:28.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:28 smithi083 ceph-mon[63709]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.9 MiB/s wr, 690 op/s; 222/1692 objects degraded (13.121%) 2024-09-18T07:32:28.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:28 smithi083 ceph-mon[63709]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:32:28.956 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:32:28.956 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:32:28.956 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:32:28.956 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:32:28.956 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:32:28.956 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:32:28.956 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:32:28.956 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 4 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:32:28.957 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:32:29.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:28 smithi096 ceph-mon[49483]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:32:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:28 smithi096 ceph-mon[49483]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.9 MiB/s wr, 690 op/s; 222/1692 objects degraded (13.121%) 2024-09-18T07:32:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:28 smithi096 ceph-mon[49483]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:32:29.191 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:28 smithi083 podman[68546]: 2024-09-18 07:32:28.86276445 +0000 UTC m=+0.021624687 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:32:29.192 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:29 smithi083 podman[68546]: 2024-09-18 07:32:29.000548385 +0000 UTC m=+0.159408612 container create 614f60ad38ff05050a86bc0f0112589953102c001376346d656254104a77661f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:32:29.392 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "6/23 daemons upgraded", 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:32:29.393 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:32:29.442 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:29 smithi083 podman[68546]: 2024-09-18 07:32:29.362228577 +0000 UTC m=+0.521088812 container init 614f60ad38ff05050a86bc0f0112589953102c001376346d656254104a77661f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS) 2024-09-18T07:32:29.443 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:29 smithi083 podman[68546]: 2024-09-18 07:32:29.367149453 +0000 UTC m=+0.526009678 container start 614f60ad38ff05050a86bc0f0112589953102c001376346d656254104a77661f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True) 2024-09-18T07:32:29.443 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:29 smithi083 ceph-osd[68586]: -- 172.21.15.83:0/3381242094 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56189d4c81e0 con 0x56189d454000 2024-09-18T07:32:29.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 162012 10.49 MB/sec execute 278 sec latency 80.786 ms 2024-09-18T07:32:29.693 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:29 smithi083 bash[68546]: 614f60ad38ff05050a86bc0f0112589953102c001376346d656254104a77661f 2024-09-18T07:32:29.694 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:29 smithi083 systemd[1]: Started Ceph osd.0 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:32:29.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:29 smithi083 ceph-mon[63709]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:32:29.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:29 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/2566264991' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:29.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:29 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:29.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:29 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:29.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:29 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:32:30.024 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 222/1692 objects degraded (13.121%), 29 pgs degraded 2024-09-18T07:32:30.024 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-18T07:32:30.024 INFO:teuthology.orchestra.run.smithi083.stdout: osd.0 (root=default,host=smithi083) is down 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 222/1692 objects degraded (13.121%), 29 pgs degraded 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-18T07:32:30.025 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1f is active+undersized+degraded, acting [2,4] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-18T07:32:30.026 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-18T07:32:30.027 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-18T07:32:30.027 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-18T07:32:30.027 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-18T07:32:30.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:29 smithi096 ceph-mon[49483]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:32:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:29 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/2566264991' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:29 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:29 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:29 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:32:30.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 162419 10.49 MB/sec execute 279 sec latency 44.615 ms 2024-09-18T07:32:30.714 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:30 smithi083 ceph-mon[63709]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:32:30.715 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:30 smithi083 ceph-mon[63709]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 616 op/s; 222/1692 objects degraded (13.121%) 2024-09-18T07:32:30.715 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:30 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/573140088' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:32:31.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:30 smithi096 ceph-mon[49483]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:32:31.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:30 smithi096 ceph-mon[49483]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 355 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 616 op/s; 222/1692 objects degraded (13.121%) 2024-09-18T07:32:31.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:30 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/573140088' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:32:31.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 162897 10.48 MB/sec execute 280 sec latency 55.592 ms 2024-09-18T07:32:31.979 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:31.980 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:32:31.980 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:31 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[68582]: 2024-09-18T07:32:31.751+0000 7f997fe68640 -1 Falling back to public interface 2024-09-18T07:32:32.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:32.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:32:32.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 163119 10.46 MB/sec execute 281 sec latency 37.665 ms 2024-09-18T07:32:32.988 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:32 smithi083 ceph-mon[63709]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.8 MiB/s wr, 716 op/s; 198/1599 objects degraded (12.383%) 2024-09-18T07:32:32.988 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:32 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 198/1599 objects degraded (12.383%), 29 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:33.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:32 smithi096 ceph-mon[49483]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.8 MiB/s wr, 716 op/s; 198/1599 objects degraded (12.383%) 2024-09-18T07:32:33.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:32 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 198/1599 objects degraded (12.383%), 29 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:33.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 163539 10.49 MB/sec execute 282 sec latency 34.179 ms 2024-09-18T07:32:34.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 163852 10.48 MB/sec execute 283 sec latency 39.198 ms 2024-09-18T07:32:35.056 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:34 smithi083 ceph-mon[63709]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.1 MiB/s wr, 506 op/s; 198/1599 objects degraded (12.383%) 2024-09-18T07:32:35.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:34 smithi096 ceph-mon[49483]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.1 MiB/s wr, 506 op/s; 198/1599 objects degraded (12.383%) 2024-09-18T07:32:35.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 164170 10.50 MB/sec execute 284 sec latency 46.838 ms 2024-09-18T07:32:36.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 164319 10.48 MB/sec execute 285 sec latency 75.194 ms 2024-09-18T07:32:37.102 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:36 smithi083 ceph-mon[63709]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 494 op/s; 198/1599 objects degraded (12.383%) 2024-09-18T07:32:37.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:36 smithi096 ceph-mon[49483]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 494 op/s; 198/1599 objects degraded (12.383%) 2024-09-18T07:32:37.544 INFO:teuthology.orchestra.run.smithi083.stdout: 5 164502 10.46 MB/sec execute 286 sec latency 73.432 ms 2024-09-18T07:32:38.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 164816 10.43 MB/sec execute 287 sec latency 60.723 ms 2024-09-18T07:32:38.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:38 smithi083 ceph-mon[63709]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.2 MiB/s wr, 675 op/s; 206/1638 objects degraded (12.576%) 2024-09-18T07:32:38.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:38.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:38.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:38 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 206/1638 objects degraded (12.576%), 29 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:38.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:38.893 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:39.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:38 smithi096 ceph-mon[49483]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.2 MiB/s wr, 675 op/s; 206/1638 objects degraded (12.576%) 2024-09-18T07:32:39.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:39.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:39.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:38 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 206/1638 objects degraded (12.576%), 29 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:39.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:39.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:39.152 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:38 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[68582]: 2024-09-18T07:32:38.996+0000 7f997fe68640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-18T07:32:39.546 INFO:teuthology.orchestra.run.smithi083.stdout: 5 165070 10.40 MB/sec execute 288 sec latency 91.391 ms 2024-09-18T07:32:40.161 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:40 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[68582]: 2024-09-18T07:32:40.093+0000 7f997fe68640 -1 osd.0 73 log_to_monitors true 2024-09-18T07:32:40.546 INFO:teuthology.orchestra.run.smithi083.stdout: 5 165412 10.40 MB/sec execute 289 sec latency 35.918 ms 2024-09-18T07:32:40.916 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:32:40 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[68582]: 2024-09-18T07:32:40.905+0000 7f9977c12640 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:32:40.917 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:40 smithi083 ceph-mon[63709]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 206/1638 objects degraded (12.576%) 2024-09-18T07:32:40.917 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:40 smithi083 ceph-mon[63709]: from='osd.0 [v2:172.21.15.83:6802/1121413804,v1:172.21.15.83:6803/1121413804]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-18T07:32:41.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:40 smithi096 ceph-mon[49483]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 206/1638 objects degraded (12.576%) 2024-09-18T07:32:41.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:40 smithi096 ceph-mon[49483]: from='osd.0 [v2:172.21.15.83:6802/1121413804,v1:172.21.15.83:6803/1121413804]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-18T07:32:41.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 165869 10.42 MB/sec execute 290 sec latency 59.128 ms 2024-09-18T07:32:41.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:41 smithi083 ceph-mon[63709]: from='osd.0 [v2:172.21.15.83:6802/1121413804,v1:172.21.15.83:6803/1121413804]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-18T07:32:41.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:41 smithi083 ceph-mon[63709]: osdmap e76: 8 total, 7 up, 8 in 2024-09-18T07:32:41.939 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:41 smithi083 ceph-mon[63709]: from='osd.0 [v2:172.21.15.83:6802/1121413804,v1:172.21.15.83:6803/1121413804]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:32:42.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:41 smithi096 ceph-mon[49483]: from='osd.0 [v2:172.21.15.83:6802/1121413804,v1:172.21.15.83:6803/1121413804]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-18T07:32:42.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:41 smithi096 ceph-mon[49483]: osdmap e76: 8 total, 7 up, 8 in 2024-09-18T07:32:42.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:41 smithi096 ceph-mon[49483]: from='osd.0 [v2:172.21.15.83:6802/1121413804,v1:172.21.15.83:6803/1121413804]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:32:42.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 166080 10.39 MB/sec execute 291 sec latency 661.500 ms 2024-09-18T07:32:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:42 smithi083 ceph-mon[63709]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 459 op/s; 201/1566 objects degraded (12.835%) 2024-09-18T07:32:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:42 smithi083 ceph-mon[63709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:32:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:42 smithi083 ceph-mon[63709]: osd.0 [v2:172.21.15.83:6802/1121413804,v1:172.21.15.83:6803/1121413804] boot 2024-09-18T07:32:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:42 smithi083 ceph-mon[63709]: osdmap e77: 8 total, 8 up, 8 in 2024-09-18T07:32:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-18T07:32:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:42 smithi083 ceph-mon[63709]: osdmap e78: 8 total, 8 up, 8 in 2024-09-18T07:32:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:42 smithi096 ceph-mon[49483]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 459 op/s; 201/1566 objects degraded (12.835%) 2024-09-18T07:32:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:42 smithi096 ceph-mon[49483]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:32:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:42 smithi096 ceph-mon[49483]: osd.0 [v2:172.21.15.83:6802/1121413804,v1:172.21.15.83:6803/1121413804] boot 2024-09-18T07:32:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:42 smithi096 ceph-mon[49483]: osdmap e77: 8 total, 8 up, 8 in 2024-09-18T07:32:43.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-18T07:32:43.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:42 smithi096 ceph-mon[49483]: osdmap e78: 8 total, 8 up, 8 in 2024-09-18T07:32:43.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 166080 10.35 MB/sec execute 292 sec latency 1661.546 ms 2024-09-18T07:32:44.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 166458 10.34 MB/sec execute 293 sec latency 1744.345 ms 2024-09-18T07:32:44.672 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:44 smithi083 ceph-mon[63709]: osdmap e79: 8 total, 8 up, 8 in 2024-09-18T07:32:44.672 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:44 smithi083 ceph-mon[63709]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 606 op/s; 201/1566 objects degraded (12.835%) 2024-09-18T07:32:44.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:44 smithi096 ceph-mon[49483]: osdmap e79: 8 total, 8 up, 8 in 2024-09-18T07:32:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:44 smithi096 ceph-mon[49483]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 606 op/s; 201/1566 objects degraded (12.835%) 2024-09-18T07:32:45.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 166606 10.32 MB/sec execute 294 sec latency 824.323 ms 2024-09-18T07:32:45.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:45 smithi083 ceph-mon[63709]: osdmap e80: 8 total, 8 up, 8 in 2024-09-18T07:32:45.685 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:45 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:45.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:45 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:45.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:45 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:32:45.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:45 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:32:45.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:45 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:45.686 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:45 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:45.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:45 smithi096 ceph-mon[49483]: osdmap e80: 8 total, 8 up, 8 in 2024-09-18T07:32:45.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:45 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:45.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:45 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:45.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:45 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:32:45.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:45 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:32:45.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:45 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:45.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:45 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:46.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 166854 10.30 MB/sec execute 295 sec latency 836.227 ms 2024-09-18T07:32:46.717 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: osdmap e81: 8 total, 8 up, 8 in 2024-09-18T07:32:46.717 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 201/1566 objects degraded (12.835%) 2024-09-18T07:32:46.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:32:46.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:46.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:46.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:46.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-18T07:32:46.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-18T07:32:46.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-18T07:32:46.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:46.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: osdmap e81: 8 total, 8 up, 8 in 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 201/1566 objects degraded (12.835%) 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-18T07:32:46.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:32:46.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:32:47.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 167199 10.32 MB/sec execute 296 sec latency 879.690 ms 2024-09-18T07:32:47.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:47 smithi083 ceph-mon[63709]: osdmap e82: 8 total, 8 up, 8 in 2024-09-18T07:32:47.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:47 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 201/1566 objects degraded (12.835%), 29 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:47.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:47 smithi096 ceph-mon[49483]: osdmap e82: 8 total, 8 up, 8 in 2024-09-18T07:32:47.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:47 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 201/1566 objects degraded (12.835%), 29 pgs degraded (PG_DEGRADED) 2024-09-18T07:32:48.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 167635 10.34 MB/sec execute 297 sec latency 890.220 ms 2024-09-18T07:32:48.593 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:48 smithi083 ceph-mon[63709]: pgmap v73: 97 pgs: 1 peering, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.3 MiB/s wr, 355 op/s; 174/1545 objects degraded (11.262%); 1.0 MiB/s, 35 objects/s recovering 2024-09-18T07:32:48.593 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:48 smithi083 ceph-mon[63709]: osdmap e83: 8 total, 8 up, 8 in 2024-09-18T07:32:48.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:48 smithi096 ceph-mon[49483]: pgmap v73: 97 pgs: 1 peering, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.3 MiB/s wr, 355 op/s; 174/1545 objects degraded (11.262%); 1.0 MiB/s, 35 objects/s recovering 2024-09-18T07:32:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:48 smithi096 ceph-mon[49483]: osdmap e83: 8 total, 8 up, 8 in 2024-09-18T07:32:49.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 167849 10.32 MB/sec execute 298 sec latency 990.887 ms 2024-09-18T07:32:49.605 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:49 smithi083 ceph-mon[63709]: osdmap e84: 8 total, 8 up, 8 in 2024-09-18T07:32:49.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:49 smithi096 ceph-mon[49483]: osdmap e84: 8 total, 8 up, 8 in 2024-09-18T07:32:50.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 168167 10.32 MB/sec execute 299 sec latency 1044.697 ms 2024-09-18T07:32:50.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:50 smithi083 ceph-mon[63709]: pgmap v76: 97 pgs: 1 peering, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.3 MiB/s wr, 356 op/s; 174/1545 objects degraded (11.262%); 1.0 MiB/s, 35 objects/s recovering 2024-09-18T07:32:50.666 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:50 smithi083 ceph-mon[63709]: osdmap e85: 8 total, 8 up, 8 in 2024-09-18T07:32:50.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:50 smithi096 ceph-mon[49483]: pgmap v76: 97 pgs: 1 peering, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.3 MiB/s wr, 356 op/s; 174/1545 objects degraded (11.262%); 1.0 MiB/s, 35 objects/s recovering 2024-09-18T07:32:50.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:50 smithi096 ceph-mon[49483]: osdmap e85: 8 total, 8 up, 8 in 2024-09-18T07:32:51.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 168406 10.30 MB/sec execute 300 sec latency 986.797 ms 2024-09-18T07:32:51.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:51 smithi083 ceph-mon[63709]: osdmap e86: 8 total, 8 up, 8 in 2024-09-18T07:32:51.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:51 smithi096 ceph-mon[49483]: osdmap e86: 8 total, 8 up, 8 in 2024-09-18T07:32:52.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 168758 10.29 MB/sec execute 301 sec latency 679.170 ms 2024-09-18T07:32:52.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:52 smithi083 ceph-mon[63709]: pgmap v79: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 574 op/s; 0 B/s, 48 objects/s recovering 2024-09-18T07:32:52.658 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:52 smithi083 ceph-mon[63709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 174/1545 objects degraded (11.262%), 7 pgs degraded) 2024-09-18T07:32:52.658 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:52 smithi083 ceph-mon[63709]: Cluster is now healthy 2024-09-18T07:32:52.658 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:52 smithi083 ceph-mon[63709]: osdmap e87: 8 total, 8 up, 8 in 2024-09-18T07:32:52.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:52 smithi096 ceph-mon[49483]: pgmap v79: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 574 op/s; 0 B/s, 48 objects/s recovering 2024-09-18T07:32:52.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:52 smithi096 ceph-mon[49483]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 174/1545 objects degraded (11.262%), 7 pgs degraded) 2024-09-18T07:32:52.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:52 smithi096 ceph-mon[49483]: Cluster is now healthy 2024-09-18T07:32:52.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:52 smithi096 ceph-mon[49483]: osdmap e87: 8 total, 8 up, 8 in 2024-09-18T07:32:53.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 169081 10.27 MB/sec execute 302 sec latency 701.012 ms 2024-09-18T07:32:53.705 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:53 smithi083 ceph-mon[63709]: osdmap e88: 8 total, 8 up, 8 in 2024-09-18T07:32:53.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:53 smithi096 ceph-mon[49483]: osdmap e88: 8 total, 8 up, 8 in 2024-09-18T07:32:54.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 169450 10.27 MB/sec execute 303 sec latency 35.599 ms 2024-09-18T07:32:54.726 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:54 smithi083 ceph-mon[63709]: pgmap v82: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 574 op/s; 0 B/s, 48 objects/s recovering 2024-09-18T07:32:54.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:54 smithi096 ceph-mon[49483]: pgmap v82: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 574 op/s; 0 B/s, 48 objects/s recovering 2024-09-18T07:32:55.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 169779 10.26 MB/sec execute 304 sec latency 61.763 ms 2024-09-18T07:32:56.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 170208 10.27 MB/sec execute 305 sec latency 34.129 ms 2024-09-18T07:32:56.769 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:56 smithi083 ceph-mon[63709]: pgmap v83: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 384 op/s; 0 B/s, 32 objects/s recovering 2024-09-18T07:32:56.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:56 smithi096 ceph-mon[49483]: pgmap v83: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 384 op/s; 0 B/s, 32 objects/s recovering 2024-09-18T07:32:57.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 170503 10.27 MB/sec execute 306 sec latency 42.385 ms 2024-09-18T07:32:58.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 171145 10.33 MB/sec execute 307 sec latency 22.746 ms 2024-09-18T07:32:58.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:32:58 smithi083 ceph-mon[63709]: pgmap v84: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 694 op/s; 0 B/s, 27 objects/s recovering 2024-09-18T07:32:58.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:32:58 smithi096 ceph-mon[49483]: pgmap v84: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 694 op/s; 0 B/s, 27 objects/s recovering 2024-09-18T07:32:59.547 INFO:teuthology.orchestra.run.smithi083.stdout: 5 171594 10.32 MB/sec execute 308 sec latency 27.680 ms 2024-09-18T07:33:00.549 INFO:teuthology.orchestra.run.smithi083.stdout: 5 172063 10.31 MB/sec execute 309 sec latency 37.418 ms 2024-09-18T07:33:00.624 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:33:00.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:00 smithi083 ceph-mon[63709]: pgmap v85: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.7 MiB/s wr, 318 op/s 2024-09-18T07:33:00.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:00 smithi096 ceph-mon[49483]: pgmap v85: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.7 MiB/s wr, 318 op/s 2024-09-18T07:33:01.549 INFO:teuthology.orchestra.run.smithi083.stdout: 5 172499 10.31 MB/sec execute 310 sec latency 43.279 ms 2024-09-18T07:33:01.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-18T07:33:01.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:01 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-18T07:33:01.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:01 smithi083 ceph-mon[63709]: Upgrade: osd.1 is safe to restart 2024-09-18T07:33:01.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:01 smithi083 ceph-mon[63709]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:01.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:01.590 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:33:01.784 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:33:01.784 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (4m) 23s ago 13m 18.8M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:33:01.784 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (72s) 23s ago 13m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:33:01.784 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (66s) 62s ago 13m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:33:01.785 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (4m) 23s ago 13m 46.5M - dad864ee21e9 06ff04a18f50 2024-09-18T07:33:01.785 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (7m) 23s ago 7m 40.3M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:33:01.785 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (7m) 62s ago 7m 16.0M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:33:01.785 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (3m) 23s ago 14m 511M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:33:01.785 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (2m) 62s ago 13m 453M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:33:01.785 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (2m) 23s ago 15m 46.6M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:33:01.786 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (119s) 62s ago 12m 35.8M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:33:01.786 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (7m) 23s ago 7m 268M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:33:01.786 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (7m) 62s ago 7m 90.1M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:33:01.786 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (4m) 23s ago 13m 10.2M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:33:01.786 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (3m) 62s ago 12m 12.2M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:33:01.786 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (32s) 23s ago 12m 11.7M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:33:01.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11m) 23s ago 11m 489M 1936M 17.2.0 e1d6a67b021e ee54ca1eff60 2024-09-18T07:33:01.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11m) 23s ago 11m 392M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:33:01.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10m) 23s ago 10m 508M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:33:01.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (10m) 62s ago 10m 507M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:33:01.787 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (9m) 62s ago 9m 567M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:33:01.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (9m) 62s ago 9m 502M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:33:01.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (9m) 62s ago 9m 388M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:33:01.788 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (2m) 23s ago 12m 48.7M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:33:01.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-18T07:33:01.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:01 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-18T07:33:01.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:01 smithi096 ceph-mon[49483]: Upgrade: osd.1 is safe to restart 2024-09-18T07:33:01.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:01 smithi096 ceph-mon[49483]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:01.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:01.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:33:02.522 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:33:02.522 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:33:02.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:33:02.522 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:33:02.522 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:33:02.522 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 1 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 5 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:33:02.523 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:33:02.550 INFO:teuthology.orchestra.run.smithi083.stdout: 5 172976 10.31 MB/sec execute 311 sec latency 35.383 ms 2024-09-18T07:33:03.104 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:02 smithi083 ceph-mon[63709]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:03.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:02 smithi083 ceph-mon[63709]: pgmap v86: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 583 op/s 2024-09-18T07:33:03.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:02 smithi083 ceph-mon[63709]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:03.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:02 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/3380528251' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:03.115 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "7/23 daemons upgraded", 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:33:03.116 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:33:03.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:02 smithi096 ceph-mon[49483]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:03.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:02 smithi096 ceph-mon[49483]: pgmap v86: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 583 op/s 2024-09-18T07:33:03.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:02 smithi096 ceph-mon[49483]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:03.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:02 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/3380528251' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:03.555 INFO:teuthology.orchestra.run.smithi083.stdout: 5 173391 10.31 MB/sec execute 312 sec latency 42.740 ms 2024-09-18T07:33:03.837 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:33:04.372 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:04 smithi083 ceph-mon[63709]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:04.373 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:04 smithi083 ceph-mon[63709]: Upgrade: Updating osd.1 2024-09-18T07:33:04.373 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:04.373 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-18T07:33:04.373 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:04 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:33:04.373 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:04 smithi083 ceph-mon[63709]: Deploying daemon osd.1 on smithi083 2024-09-18T07:33:04.373 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:04 smithi083 ceph-mon[63709]: pgmap v87: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 549 op/s 2024-09-18T07:33:04.373 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:04 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1811864497' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:33:04.555 INFO:teuthology.orchestra.run.smithi083.stdout: 5 173855 10.33 MB/sec execute 313 sec latency 31.361 ms 2024-09-18T07:33:04.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:04 smithi096 ceph-mon[49483]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:04 smithi096 ceph-mon[49483]: Upgrade: Updating osd.1 2024-09-18T07:33:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-18T07:33:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:04 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:33:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:04 smithi096 ceph-mon[49483]: Deploying daemon osd.1 on smithi083 2024-09-18T07:33:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:04 smithi096 ceph-mon[49483]: pgmap v87: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 549 op/s 2024-09-18T07:33:04.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:04 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1811864497' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:33:05.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 174441 10.37 MB/sec execute 314 sec latency 26.922 ms 2024-09-18T07:33:06.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 174848 10.38 MB/sec execute 315 sec latency 45.408 ms 2024-09-18T07:33:07.165 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:06 smithi083 ceph-mon[63709]: pgmap v88: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 502 op/s 2024-09-18T07:33:07.165 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:06 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:07.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:06 smithi096 ceph-mon[49483]: pgmap v88: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 502 op/s 2024-09-18T07:33:07.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:06 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:07.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 175154 10.36 MB/sec execute 316 sec latency 42.527 ms 2024-09-18T07:33:08.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 175465 10.35 MB/sec execute 317 sec latency 47.829 ms 2024-09-18T07:33:08.679 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:08 smithi083 systemd[1]: Stopping Ceph osd.1 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:33:08.932 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:08 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[38155]: 2024-09-18T07:33:08.916+0000 7fe279b95700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:33:08.932 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:08 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[38155]: 2024-09-18T07:33:08.916+0000 7fe279b95700 -1 osd.1 88 *** Got signal Terminated *** 2024-09-18T07:33:08.932 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:08 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[38155]: 2024-09-18T07:33:08.916+0000 7fe279b95700 -1 osd.1 88 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:33:09.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:08 smithi083 ceph-mon[63709]: pgmap v89: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 773 op/s 2024-09-18T07:33:09.184 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:08 smithi083 ceph-mon[63709]: osd.1 marked itself down and dead 2024-09-18T07:33:09.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:08 smithi096 ceph-mon[49483]: pgmap v89: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 773 op/s 2024-09-18T07:33:09.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:08 smithi096 ceph-mon[49483]: osd.1 marked itself down and dead 2024-09-18T07:33:09.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 175643 10.32 MB/sec execute 318 sec latency 528.637 ms 2024-09-18T07:33:09.894 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:09 smithi083 podman[70486]: 2024-09-18 07:33:09.627182118 +0000 UTC m=+0.848471784 container died ee54ca1eff607d80ada9804d6fa3ae2881e7f5423bd1582de1bede3ed69fd8cd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, RELEASE=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-18T07:33:10.145 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:10 smithi083 podman[70486]: 2024-09-18 07:33:10.002043393 +0000 UTC m=+1.223333065 container cleanup ee54ca1eff607d80ada9804d6fa3ae2881e7f5423bd1582de1bede3ed69fd8cd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, distribution-scope=public, ceph=True, release=754, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, version=8, io.openshift.expose-services=) 2024-09-18T07:33:10.145 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:10 smithi083 bash[70486]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1 2024-09-18T07:33:10.146 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:09 smithi083 ceph-mon[63709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:33:10.146 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:09 smithi083 ceph-mon[63709]: osdmap e89: 8 total, 7 up, 8 in 2024-09-18T07:33:10.272 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:09 smithi096 ceph-mon[49483]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:33:10.272 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:09 smithi096 ceph-mon[49483]: osdmap e89: 8 total, 7 up, 8 in 2024-09-18T07:33:10.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 176020 10.32 MB/sec execute 319 sec latency 993.345 ms 2024-09-18T07:33:10.648 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:10 smithi083 podman[70501]: 2024-09-18 07:33:10.419718975 +0000 UTC m=+0.789114841 container remove ee54ca1eff607d80ada9804d6fa3ae2881e7f5423bd1582de1bede3ed69fd8cd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.tags=base centos centos-stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, ceph=True, maintainer=Guillaume Abrioux , name=centos-stream, RELEASE=HEAD, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc.) 2024-09-18T07:33:11.024 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:10 smithi083 ceph-mon[63709]: pgmap v91: 97 pgs: 13 stale+active+clean, 84 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.3 MiB/s wr, 673 op/s 2024-09-18T07:33:11.024 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:10 smithi083 ceph-mon[63709]: osdmap e90: 8 total, 7 up, 8 in 2024-09-18T07:33:11.149 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:11 smithi083 podman[70581]: 2024-09-18 07:33:11.010214217 +0000 UTC m=+0.025764353 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:33:11.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:10 smithi096 ceph-mon[49483]: pgmap v91: 97 pgs: 13 stale+active+clean, 84 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.3 MiB/s wr, 673 op/s 2024-09-18T07:33:11.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:10 smithi096 ceph-mon[49483]: osdmap e90: 8 total, 7 up, 8 in 2024-09-18T07:33:11.403 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:11 smithi083 podman[70581]: 2024-09-18 07:33:11.171484866 +0000 UTC m=+0.187034974 container create 7ee060bbb5840055ecfbdff722cc65a5d8b7358513c3c415fec89c156834db10 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True) 2024-09-18T07:33:11.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 176748 10.34 MB/sec execute 320 sec latency 27.302 ms 2024-09-18T07:33:11.656 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:11 smithi083 podman[70581]: 2024-09-18 07:33:11.602274892 +0000 UTC m=+0.617825005 container init 7ee060bbb5840055ecfbdff722cc65a5d8b7358513c3c415fec89c156834db10 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-18T07:33:11.656 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:11 smithi083 podman[70581]: 2024-09-18 07:33:11.613533753 +0000 UTC m=+0.629083862 container start 7ee060bbb5840055ecfbdff722cc65a5d8b7358513c3c415fec89c156834db10 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:33:12.160 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:12 smithi083 ceph-mon[63709]: Health check failed: Degraded data redundancy: 143/1611 objects degraded (8.876%), 19 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:12.161 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:11 smithi083 podman[70581]: 2024-09-18 07:33:11.918818406 +0000 UTC m=+0.934368514 container attach 7ee060bbb5840055ecfbdff722cc65a5d8b7358513c3c415fec89c156834db10 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-18T07:33:12.161 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:11 smithi083 podman[70604]: 2024-09-18 07:33:11.997960608 +0000 UTC m=+0.020652917 container died 7ee060bbb5840055ecfbdff722cc65a5d8b7358513c3c415fec89c156834db10 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD) 2024-09-18T07:33:12.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:12 smithi096 ceph-mon[49483]: Health check failed: Degraded data redundancy: 143/1611 objects degraded (8.876%), 19 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:12.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 177349 10.36 MB/sec execute 321 sec latency 42.184 ms 2024-09-18T07:33:13.164 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:13 smithi083 ceph-mon[63709]: pgmap v93: 97 pgs: 14 active+undersized, 2 stale+active+clean, 19 active+undersized+degraded, 62 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 737 op/s; 143/1611 objects degraded (8.876%) 2024-09-18T07:33:13.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:13 smithi096 ceph-mon[49483]: pgmap v93: 97 pgs: 14 active+undersized, 2 stale+active+clean, 19 active+undersized+degraded, 62 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 737 op/s; 143/1611 objects degraded (8.876%) 2024-09-18T07:33:13.416 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:13 smithi083 podman[70604]: 2024-09-18 07:33:13.237569572 +0000 UTC m=+1.260261890 container remove 7ee060bbb5840055ecfbdff722cc65a5d8b7358513c3c415fec89c156834db10 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-18T07:33:13.416 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:13 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-18T07:33:13.416 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:13 smithi083 systemd[1]: Stopped Ceph osd.1 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:33:13.416 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:13 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.1.service: Consumed 3min 2.056s CPU time. 2024-09-18T07:33:13.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 178260 10.44 MB/sec execute 322 sec latency 23.927 ms 2024-09-18T07:33:13.921 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:13 smithi083 systemd[1]: Starting Ceph osd.1 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:33:14.175 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:14 smithi083 ceph-mon[63709]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 769 op/s; 180/1623 objects degraded (11.091%) 2024-09-18T07:33:14.176 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:13 smithi083 podman[70697]: 2024-09-18 07:33:13.830880994 +0000 UTC m=+0.021425862 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:33:14.176 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:13 smithi083 podman[70697]: 2024-09-18 07:33:13.955420093 +0000 UTC m=+0.145964953 container create 866734e9e8d0eeb260fae3c993b770caa88f0694c2eedc50aaf66aac44db094e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-18T07:33:14.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:14 smithi096 ceph-mon[49483]: pgmap v94: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 769 op/s; 180/1623 objects degraded (11.091%) 2024-09-18T07:33:14.426 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:14 smithi083 podman[70697]: 2024-09-18 07:33:14.341389628 +0000 UTC m=+0.531934489 container init 866734e9e8d0eeb260fae3c993b770caa88f0694c2eedc50aaf66aac44db094e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-18T07:33:14.427 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:14 smithi083 podman[70697]: 2024-09-18 07:33:14.354274694 +0000 UTC m=+0.544819552 container start 866734e9e8d0eeb260fae3c993b770caa88f0694c2eedc50aaf66aac44db094e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0) 2024-09-18T07:33:14.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 178624 10.43 MB/sec execute 323 sec latency 45.468 ms 2024-09-18T07:33:14.681 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:14 smithi083 podman[70697]: 2024-09-18 07:33:14.474785006 +0000 UTC m=+0.665329861 container attach 866734e9e8d0eeb260fae3c993b770caa88f0694c2eedc50aaf66aac44db094e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5) 2024-09-18T07:33:14.682 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:14 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate[70711]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:33:14.682 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:14 smithi083 bash[70697]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:33:14.682 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:14 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate[70711]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:33:14.682 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:14 smithi083 bash[70697]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:33:15.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 178953 10.41 MB/sec execute 324 sec latency 56.692 ms 2024-09-18T07:33:16.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 179524 10.42 MB/sec execute 325 sec latency 34.537 ms 2024-09-18T07:33:17.044 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:16 smithi083 ceph-mon[63709]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.9 MiB/s wr, 362 op/s; 180/1623 objects degraded (11.091%) 2024-09-18T07:33:17.045 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:17.045 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:33:17.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:16 smithi096 ceph-mon[49483]: pgmap v95: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.9 MiB/s wr, 362 op/s; 180/1623 objects degraded (11.091%) 2024-09-18T07:33:17.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:17.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:33:17.547 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate[70711]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-18T07:33:17.547 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 bash[70697]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-18T07:33:17.547 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate[70711]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-18T07:33:17.547 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 bash[70697]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-18T07:33:17.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 180400 10.44 MB/sec execute 326 sec latency 30.590 ms 2024-09-18T07:33:18.048 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:17 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 180/1623 objects degraded (11.091%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:18.049 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate[70711]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-18T07:33:18.049 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 bash[70697]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-18T07:33:18.049 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate[70711]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-18T07:33:18.049 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 bash[70697]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-18T07:33:18.049 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate[70711]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-18T07:33:18.050 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 bash[70697]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-18T07:33:18.050 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate[70711]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-18T07:33:18.050 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 bash[70697]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-18T07:33:18.050 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate[70711]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-18T07:33:18.050 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 bash[70697]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-18T07:33:18.050 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:17 smithi083 podman[70697]: 2024-09-18 07:33:17.880515595 +0000 UTC m=+4.071060459 container died 866734e9e8d0eeb260fae3c993b770caa88f0694c2eedc50aaf66aac44db094e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-18T07:33:18.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:17 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 180/1623 objects degraded (11.091%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:18.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 180914 10.47 MB/sec execute 327 sec latency 32.321 ms 2024-09-18T07:33:19.096 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:18 smithi083 ceph-mon[63709]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 810 op/s; 170/1653 objects degraded (10.284%) 2024-09-18T07:33:19.356 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:19 smithi083 podman[70890]: 2024-09-18 07:33:19.233991804 +0000 UTC m=+1.336774145 container remove 866734e9e8d0eeb260fae3c993b770caa88f0694c2eedc50aaf66aac44db094e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:33:19.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:18 smithi096 ceph-mon[49483]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 810 op/s; 170/1653 objects degraded (10.284%) 2024-09-18T07:33:19.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 181307 10.49 MB/sec execute 328 sec latency 45.397 ms 2024-09-18T07:33:19.598 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:19 smithi083 podman[70944]: 2024-09-18 07:33:19.40331193 +0000 UTC m=+0.029759771 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:33:19.598 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:19 smithi083 podman[70944]: 2024-09-18 07:33:19.526356551 +0000 UTC m=+0.152804375 container create c5493455402e5b7f26956f72bb01e9566197f7699abff7a9f56570402457c093 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0) 2024-09-18T07:33:19.864 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:19 smithi083 podman[70944]: 2024-09-18 07:33:19.820973686 +0000 UTC m=+0.447421528 container init c5493455402e5b7f26956f72bb01e9566197f7699abff7a9f56570402457c093 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:33:19.864 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:19 smithi083 podman[70944]: 2024-09-18 07:33:19.831253869 +0000 UTC m=+0.457701699 container start c5493455402e5b7f26956f72bb01e9566197f7699abff7a9f56570402457c093 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-18T07:33:20.116 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:19 smithi083 ceph-osd[70960]: -- 172.21.15.83:0/3550273001 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e0398ba1e0 con 0x55e038a55c00 2024-09-18T07:33:20.116 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:19 smithi083 bash[70944]: c5493455402e5b7f26956f72bb01e9566197f7699abff7a9f56570402457c093 2024-09-18T07:33:20.116 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:19 smithi083 systemd[1]: Started Ceph osd.1 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:33:20.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 181648 10.49 MB/sec execute 329 sec latency 33.931 ms 2024-09-18T07:33:21.181 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:21 smithi083 ceph-mon[63709]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 693 op/s; 170/1653 objects degraded (10.284%) 2024-09-18T07:33:21.182 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:21.182 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:21.182 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:33:21.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:21 smithi096 ceph-mon[49483]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 693 op/s; 170/1653 objects degraded (10.284%) 2024-09-18T07:33:21.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:21.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:21.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:33:21.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 181919 10.49 MB/sec execute 330 sec latency 96.611 ms 2024-09-18T07:33:22.187 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:21 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[70956]: 2024-09-18T07:33:21.933+0000 7f2315633640 -1 Falling back to public interface 2024-09-18T07:33:22.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 182109 10.47 MB/sec execute 331 sec latency 57.361 ms 2024-09-18T07:33:23.250 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:23 smithi083 ceph-mon[63709]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 862 op/s; 169/1674 objects degraded (10.096%) 2024-09-18T07:33:23.250 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:23 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 169/1674 objects degraded (10.096%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:23.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:23 smithi096 ceph-mon[49483]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 862 op/s; 169/1674 objects degraded (10.096%) 2024-09-18T07:33:23.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:23 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 169/1674 objects degraded (10.096%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:23.556 INFO:teuthology.orchestra.run.smithi083.stdout: 5 182421 10.44 MB/sec execute 332 sec latency 110.532 ms 2024-09-18T07:33:24.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 182681 10.43 MB/sec execute 333 sec latency 62.108 ms 2024-09-18T07:33:25.075 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:25 smithi083 ceph-mon[63709]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.1 MiB/s wr, 631 op/s; 167/1692 objects degraded (9.870%) 2024-09-18T07:33:25.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:25 smithi096 ceph-mon[49483]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.1 MiB/s wr, 631 op/s; 167/1692 objects degraded (9.870%) 2024-09-18T07:33:25.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 182979 10.42 MB/sec execute 334 sec latency 53.790 ms 2024-09-18T07:33:26.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 183354 10.41 MB/sec execute 335 sec latency 77.297 ms 2024-09-18T07:33:27.117 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:27 smithi083 ceph-mon[63709]: pgmap v100: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 609 op/s; 167/1692 objects degraded (9.870%) 2024-09-18T07:33:27.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:27 smithi096 ceph-mon[49483]: pgmap v100: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 609 op/s; 167/1692 objects degraded (9.870%) 2024-09-18T07:33:27.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 183712 10.40 MB/sec execute 336 sec latency 49.784 ms 2024-09-18T07:33:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:28 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 167/1692 objects degraded (9.870%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:28.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:28 smithi083 ceph-mon[63709]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 806 op/s; 157/1506 objects degraded (10.425%) 2024-09-18T07:33:28.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:28 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 167/1692 objects degraded (9.870%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:28.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:28 smithi096 ceph-mon[49483]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 806 op/s; 157/1506 objects degraded (10.425%) 2024-09-18T07:33:28.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 184053 10.39 MB/sec execute 337 sec latency 80.398 ms 2024-09-18T07:33:29.200 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:29 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[70956]: 2024-09-18T07:33:29.180+0000 7f2315633640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-18T07:33:29.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 184403 10.40 MB/sec execute 338 sec latency 42.708 ms 2024-09-18T07:33:29.976 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:29 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:29.976 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:29 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:29.976 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:29 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:29.976 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:29 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:30.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:29 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:29 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:29 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:29 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:30.489 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:30 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[70956]: 2024-09-18T07:33:30.302+0000 7f2315633640 -1 osd.1 88 log_to_monitors true 2024-09-18T07:33:30.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 184811 10.42 MB/sec execute 339 sec latency 34.492 ms 2024-09-18T07:33:31.032 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:30 smithi083 ceph-mon[63709]: pgmap v102: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 470 op/s; 157/1506 objects degraded (10.425%) 2024-09-18T07:33:31.032 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:30 smithi083 ceph-mon[63709]: from='osd.1 [v2:172.21.15.83:6810/3246340873,v1:172.21.15.83:6811/3246340873]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-18T07:33:31.032 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:30 smithi083 ceph-mon[63709]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-18T07:33:31.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:30 smithi096 ceph-mon[49483]: pgmap v102: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 470 op/s; 157/1506 objects degraded (10.425%) 2024-09-18T07:33:31.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:30 smithi096 ceph-mon[49483]: from='osd.1 [v2:172.21.15.83:6810/3246340873,v1:172.21.15.83:6811/3246340873]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-18T07:33:31.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:30 smithi096 ceph-mon[49483]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-18T07:33:31.532 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:33:31 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[70956]: 2024-09-18T07:33:31.335+0000 7f23093d5640 -1 osd.1 88 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:33:31.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 185084 10.43 MB/sec execute 340 sec latency 194.938 ms 2024-09-18T07:33:32.034 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:31 smithi083 ceph-mon[63709]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-18T07:33:32.035 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:31 smithi083 ceph-mon[63709]: osdmap e91: 8 total, 7 up, 8 in 2024-09-18T07:33:32.035 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:31 smithi083 ceph-mon[63709]: from='osd.1 [v2:172.21.15.83:6810/3246340873,v1:172.21.15.83:6811/3246340873]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:33:32.035 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:31 smithi083 ceph-mon[63709]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:33:32.035 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:33:32.035 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:31 smithi083 ceph-mon[63709]: from='osd.1 ' entity='osd.1' 2024-09-18T07:33:32.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:31 smithi096 ceph-mon[49483]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-18T07:33:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:31 smithi096 ceph-mon[49483]: osdmap e91: 8 total, 7 up, 8 in 2024-09-18T07:33:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:31 smithi096 ceph-mon[49483]: from='osd.1 [v2:172.21.15.83:6810/3246340873,v1:172.21.15.83:6811/3246340873]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:33:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:31 smithi096 ceph-mon[49483]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:33:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:33:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:31 smithi096 ceph-mon[49483]: from='osd.1 ' entity='osd.1' 2024-09-18T07:33:32.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 185329 10.42 MB/sec execute 341 sec latency 265.437 ms 2024-09-18T07:33:32.890 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:32 smithi083 ceph-mon[63709]: pgmap v104: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 380 op/s; 168/1590 objects degraded (10.566%) 2024-09-18T07:33:32.890 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:32 smithi083 ceph-mon[63709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:33:32.890 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:32 smithi083 ceph-mon[63709]: osd.1 [v2:172.21.15.83:6810/3246340873,v1:172.21.15.83:6811/3246340873] boot 2024-09-18T07:33:32.890 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:32 smithi083 ceph-mon[63709]: osdmap e92: 8 total, 8 up, 8 in 2024-09-18T07:33:32.890 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-18T07:33:32.890 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:32 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 168/1590 objects degraded (10.566%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:32.890 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:32 smithi083 ceph-mon[63709]: osdmap e93: 8 total, 8 up, 8 in 2024-09-18T07:33:33.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:32 smithi096 ceph-mon[49483]: pgmap v104: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 380 op/s; 168/1590 objects degraded (10.566%) 2024-09-18T07:33:33.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:32 smithi096 ceph-mon[49483]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:33:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:32 smithi096 ceph-mon[49483]: osd.1 [v2:172.21.15.83:6810/3246340873,v1:172.21.15.83:6811/3246340873] boot 2024-09-18T07:33:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:32 smithi096 ceph-mon[49483]: osdmap e92: 8 total, 8 up, 8 in 2024-09-18T07:33:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-18T07:33:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:32 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 168/1590 objects degraded (10.566%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:32 smithi096 ceph-mon[49483]: osdmap e93: 8 total, 8 up, 8 in 2024-09-18T07:33:33.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 185430 10.39 MB/sec execute 342 sec latency 1245.740 ms 2024-09-18T07:33:34.494 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:33:34.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 185641 10.38 MB/sec execute 343 sec latency 1311.430 ms 2024-09-18T07:33:34.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:34 smithi083 ceph-mon[63709]: osdmap e94: 8 total, 8 up, 8 in 2024-09-18T07:33:34.701 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:34 smithi083 ceph-mon[63709]: pgmap v108: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 7.0 MiB/s wr, 587 op/s; 181/1644 objects degraded (11.010%) 2024-09-18T07:33:34.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:34 smithi096 ceph-mon[49483]: osdmap e94: 8 total, 8 up, 8 in 2024-09-18T07:33:34.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:34 smithi096 ceph-mon[49483]: pgmap v108: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 7.0 MiB/s wr, 587 op/s; 181/1644 objects degraded (11.010%) 2024-09-18T07:33:35.461 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (4m) 6s ago 14m 18.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (105s) 6s ago 14m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (100s) 96s ago 13m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (4m) 6s ago 14m 44.6M - dad864ee21e9 06ff04a18f50 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (8m) 6s ago 8m 40.6M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (8m) 96s ago 8m 16.0M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (4m) 6s ago 15m 517M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (3m) 96s ago 13m 453M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (2m) 6s ago 15m 48.7M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (2m) 96s ago 13m 35.8M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:33:35.462 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (7m) 6s ago 7m 270M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (7m) 96s ago 7m 90.1M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (4m) 6s ago 13m 10.5M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (4m) 96s ago 13m 12.2M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (66s) 6s ago 12m 248M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (16s) 6s ago 12m 11.8M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11m) 6s ago 11m 452M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11m) 6s ago 11m 579M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (10m) 96s ago 10m 507M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (10m) 96s ago 10m 567M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (10m) 96s ago 10m 502M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (9m) 96s ago 9m 388M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:33:35.463 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (3m) 6s ago 13m 51.0M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:33:35.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 185790 10.35 MB/sec execute 344 sec latency 60.845 ms 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: osdmap e95: 8 total, 8 up, 8 in 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:33:35.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:35.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:35.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:35.720 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-18T07:33:35.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='client.34250 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: osdmap e95: 8 total, 8 up, 8 in 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:35.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:35.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:35.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-18T07:33:36.002 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:33:36.002 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:33:36.003 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-18T07:33:36.004 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 6 2024-09-18T07:33:36.004 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:33:36.004 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:33:36.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 186017 10.34 MB/sec execute 345 sec latency 870.973 ms 2024-09-18T07:33:36.631 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:33:36.631 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:33:36.631 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "8/23 daemons upgraded", 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:33:36.632 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:33:36.738 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:36 smithi083 ceph-mon[63709]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:36.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:36 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-18T07:33:36.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:36 smithi083 ceph-mon[63709]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-18T07:33:36.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:36 smithi083 ceph-mon[63709]: osdmap e96: 8 total, 8 up, 8 in 2024-09-18T07:33:36.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:36 smithi083 ceph-mon[63709]: pgmap v111: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 181/1644 objects degraded (11.010%) 2024-09-18T07:33:36.739 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:36 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/3340964119' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:36.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:36 smithi096 ceph-mon[49483]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:36 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-18T07:33:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:36 smithi096 ceph-mon[49483]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-18T07:33:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:36 smithi096 ceph-mon[49483]: osdmap e96: 8 total, 8 up, 8 in 2024-09-18T07:33:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:36 smithi096 ceph-mon[49483]: pgmap v111: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 181/1644 objects degraded (11.010%) 2024-09-18T07:33:36.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:36 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/3340964119' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN Degraded data redundancy: 181/1644 objects degraded (11.010%), 23 pgs degraded 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 181/1644 objects degraded (11.010%), 23 pgs degraded 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-18T07:33:37.306 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-18T07:33:37.307 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-09-18T07:33:37.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 186252 10.31 MB/sec execute 346 sec latency 914.758 ms 2024-09-18T07:33:37.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:37 smithi083 ceph-mon[63709]: osdmap e97: 8 total, 8 up, 8 in 2024-09-18T07:33:37.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:37 smithi083 ceph-mon[63709]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:37.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:37 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/329607387' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:33:37.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:37 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 181/1644 objects degraded (11.010%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:37.758 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:37 smithi083 ceph-mon[63709]: osdmap e98: 8 total, 8 up, 8 in 2024-09-18T07:33:37.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:37 smithi096 ceph-mon[49483]: osdmap e97: 8 total, 8 up, 8 in 2024-09-18T07:33:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:37 smithi096 ceph-mon[49483]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:33:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:37 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/329607387' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:33:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:37 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 181/1644 objects degraded (11.010%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:33:37.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:37 smithi096 ceph-mon[49483]: osdmap e98: 8 total, 8 up, 8 in 2024-09-18T07:33:38.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 186542 10.30 MB/sec execute 347 sec latency 751.961 ms 2024-09-18T07:33:38.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:38 smithi083 ceph-mon[63709]: pgmap v114: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.7 MiB/s wr, 379 op/s; 212/1584 objects degraded (13.384%); 2.1 MiB/s, 67 keys/s, 32 objects/s recovering 2024-09-18T07:33:38.798 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:38 smithi083 ceph-mon[63709]: osdmap e99: 8 total, 8 up, 8 in 2024-09-18T07:33:38.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:38 smithi096 ceph-mon[49483]: pgmap v114: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.7 MiB/s wr, 379 op/s; 212/1584 objects degraded (13.384%); 2.1 MiB/s, 67 keys/s, 32 objects/s recovering 2024-09-18T07:33:38.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:38 smithi096 ceph-mon[49483]: osdmap e99: 8 total, 8 up, 8 in 2024-09-18T07:33:39.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 186732 10.29 MB/sec execute 348 sec latency 803.342 ms 2024-09-18T07:33:40.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 187044 10.28 MB/sec execute 349 sec latency 710.904 ms 2024-09-18T07:33:40.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:40 smithi083 ceph-mon[63709]: pgmap v116: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.7 MiB/s wr, 379 op/s; 212/1584 objects degraded (13.384%); 2.1 MiB/s, 67 keys/s, 32 objects/s recovering 2024-09-18T07:33:40.624 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:40 smithi083 ceph-mon[63709]: osdmap e100: 8 total, 8 up, 8 in 2024-09-18T07:33:40.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:40 smithi096 ceph-mon[49483]: pgmap v116: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.7 MiB/s wr, 379 op/s; 212/1584 objects degraded (13.384%); 2.1 MiB/s, 67 keys/s, 32 objects/s recovering 2024-09-18T07:33:40.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:40 smithi096 ceph-mon[49483]: osdmap e100: 8 total, 8 up, 8 in 2024-09-18T07:33:41.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 187486 10.28 MB/sec execute 350 sec latency 367.082 ms 2024-09-18T07:33:41.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:41 smithi083 ceph-mon[63709]: osdmap e101: 8 total, 8 up, 8 in 2024-09-18T07:33:41.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:41 smithi096 ceph-mon[49483]: osdmap e101: 8 total, 8 up, 8 in 2024-09-18T07:33:42.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 187958 10.29 MB/sec execute 351 sec latency 421.171 ms 2024-09-18T07:33:42.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:42 smithi083 ceph-mon[63709]: pgmap v119: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 93 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.3 MiB/s wr, 497 op/s; 29/1542 objects degraded (1.881%); 127 KiB/s, 46 objects/s recovering 2024-09-18T07:33:42.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:42 smithi083 ceph-mon[63709]: osdmap e102: 8 total, 8 up, 8 in 2024-09-18T07:33:42.657 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:42 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 29/1542 objects degraded (1.881%), 1 pg degraded (PG_DEGRADED) 2024-09-18T07:33:42.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:42 smithi096 ceph-mon[49483]: pgmap v119: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 93 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.3 MiB/s wr, 497 op/s; 29/1542 objects degraded (1.881%); 127 KiB/s, 46 objects/s recovering 2024-09-18T07:33:42.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:42 smithi096 ceph-mon[49483]: osdmap e102: 8 total, 8 up, 8 in 2024-09-18T07:33:42.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:42 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 29/1542 objects degraded (1.881%), 1 pg degraded (PG_DEGRADED) 2024-09-18T07:33:43.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 188404 10.31 MB/sec execute 352 sec latency 34.388 ms 2024-09-18T07:33:43.667 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:43 smithi083 ceph-mon[63709]: osdmap e103: 8 total, 8 up, 8 in 2024-09-18T07:33:43.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:43 smithi096 ceph-mon[49483]: osdmap e103: 8 total, 8 up, 8 in 2024-09-18T07:33:44.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 188770 10.32 MB/sec execute 353 sec latency 57.516 ms 2024-09-18T07:33:44.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:44 smithi083 ceph-mon[63709]: pgmap v122: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 544 op/s; 127 KiB/s, 53 objects/s recovering 2024-09-18T07:33:44.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:44 smithi083 ceph-mon[63709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1542 objects degraded (1.881%), 1 pg degraded) 2024-09-18T07:33:44.694 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:44 smithi083 ceph-mon[63709]: Cluster is now healthy 2024-09-18T07:33:44.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:44 smithi096 ceph-mon[49483]: pgmap v122: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 544 op/s; 127 KiB/s, 53 objects/s recovering 2024-09-18T07:33:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:44 smithi096 ceph-mon[49483]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1542 objects degraded (1.881%), 1 pg degraded) 2024-09-18T07:33:44.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:44 smithi096 ceph-mon[49483]: Cluster is now healthy 2024-09-18T07:33:45.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 189037 10.31 MB/sec execute 354 sec latency 45.874 ms 2024-09-18T07:33:46.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 189313 10.30 MB/sec execute 355 sec latency 53.751 ms 2024-09-18T07:33:46.997 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:46 smithi083 ceph-mon[63709]: pgmap v123: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.9 MiB/s wr, 363 op/s; 85 KiB/s, 35 objects/s recovering 2024-09-18T07:33:46.997 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:46.998 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:33:47.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:46 smithi096 ceph-mon[49483]: pgmap v123: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.9 MiB/s wr, 363 op/s; 85 KiB/s, 35 objects/s recovering 2024-09-18T07:33:47.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:47.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:33:47.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 189646 10.29 MB/sec execute 356 sec latency 54.167 ms 2024-09-18T07:33:48.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 190035 10.28 MB/sec execute 357 sec latency 52.144 ms 2024-09-18T07:33:49.024 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:48 smithi083 ceph-mon[63709]: pgmap v124: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.0 MiB/s wr, 690 op/s; 73 KiB/s, 30 objects/s recovering 2024-09-18T07:33:49.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:48 smithi096 ceph-mon[49483]: pgmap v124: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.0 MiB/s wr, 690 op/s; 73 KiB/s, 30 objects/s recovering 2024-09-18T07:33:49.557 INFO:teuthology.orchestra.run.smithi083.stdout: 5 190419 10.28 MB/sec execute 358 sec latency 60.441 ms 2024-09-18T07:33:50.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 190940 10.28 MB/sec execute 359 sec latency 82.563 ms 2024-09-18T07:33:51.060 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:50 smithi083 ceph-mon[63709]: pgmap v125: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.4 MiB/s wr, 352 op/s; 0 B/s, 3 objects/s recovering 2024-09-18T07:33:51.061 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-18T07:33:51.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:50 smithi096 ceph-mon[49483]: pgmap v125: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.4 MiB/s wr, 352 op/s; 0 B/s, 3 objects/s recovering 2024-09-18T07:33:51.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-18T07:33:51.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 191358 10.29 MB/sec execute 360 sec latency 53.525 ms 2024-09-18T07:33:52.078 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:51 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-18T07:33:52.078 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:51 smithi083 ceph-mon[63709]: Upgrade: osd.2 is safe to restart 2024-09-18T07:33:52.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:51 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-18T07:33:52.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:51 smithi096 ceph-mon[49483]: Upgrade: osd.2 is safe to restart 2024-09-18T07:33:52.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 191752 10.30 MB/sec execute 361 sec latency 39.872 ms 2024-09-18T07:33:53.090 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:52 smithi083 ceph-mon[63709]: pgmap v126: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 517 op/s 2024-09-18T07:33:53.091 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:53.091 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-18T07:33:53.091 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:33:53.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:52 smithi096 ceph-mon[49483]: pgmap v126: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 517 op/s 2024-09-18T07:33:53.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:53.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-18T07:33:53.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:33:53.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 192076 10.30 MB/sec execute 362 sec latency 55.928 ms 2024-09-18T07:33:54.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:53 smithi083 ceph-mon[63709]: Upgrade: Updating osd.2 2024-09-18T07:33:54.108 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:53 smithi083 ceph-mon[63709]: Deploying daemon osd.2 on smithi083 2024-09-18T07:33:54.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:53 smithi096 ceph-mon[49483]: Upgrade: Updating osd.2 2024-09-18T07:33:54.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:53 smithi096 ceph-mon[49483]: Deploying daemon osd.2 on smithi083 2024-09-18T07:33:54.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 192444 10.31 MB/sec execute 363 sec latency 85.861 ms 2024-09-18T07:33:54.994 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:54 smithi083 ceph-mon[63709]: pgmap v127: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 490 op/s 2024-09-18T07:33:55.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:54 smithi096 ceph-mon[49483]: pgmap v127: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 490 op/s 2024-09-18T07:33:55.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 192764 10.30 MB/sec execute 364 sec latency 46.108 ms 2024-09-18T07:33:56.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 193011 10.29 MB/sec execute 365 sec latency 61.230 ms 2024-09-18T07:33:57.189 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:56 smithi083 ceph-mon[63709]: pgmap v128: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 448 op/s 2024-09-18T07:33:57.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:56 smithi096 ceph-mon[49483]: pgmap v128: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 448 op/s 2024-09-18T07:33:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:33:57.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 193236 10.27 MB/sec execute 366 sec latency 81.839 ms 2024-09-18T07:33:57.958 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:33:57 smithi083 systemd[1]: Stopping Ceph osd.2 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:33:58.208 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:33:58 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[41804]: 2024-09-18T07:33:58.197+0000 7fe3c9203700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:33:58.209 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:33:58 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[41804]: 2024-09-18T07:33:58.197+0000 7fe3c9203700 -1 osd.2 103 *** Got signal Terminated *** 2024-09-18T07:33:58.209 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:33:58 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[41804]: 2024-09-18T07:33:58.197+0000 7fe3c9203700 -1 osd.2 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:33:58.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 193569 10.26 MB/sec execute 367 sec latency 46.128 ms 2024-09-18T07:33:58.724 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:58 smithi083 ceph-mon[63709]: pgmap v129: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.2 MiB/s wr, 672 op/s 2024-09-18T07:33:58.725 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:58 smithi083 ceph-mon[63709]: osd.2 marked itself down and dead 2024-09-18T07:33:58.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:58 smithi096 ceph-mon[49483]: pgmap v129: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.2 MiB/s wr, 672 op/s 2024-09-18T07:33:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:58 smithi096 ceph-mon[49483]: osd.2 marked itself down and dead 2024-09-18T07:33:59.311 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:33:59 smithi083 podman[72873]: 2024-09-18 07:33:59.253076417 +0000 UTC m=+1.310385325 container died 361ac14ecd8ba6d1f64d60f03c5fd0fc0d88730d505b1b14df167b0780c3c0ae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.19.8, release=754, ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-type=git, architecture=x86_64, com.redhat.component=centos-stream-container, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8) 2024-09-18T07:33:59.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 193680 10.24 MB/sec execute 368 sec latency 940.062 ms 2024-09-18T07:33:59.813 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:59 smithi083 ceph-mon[63709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:33:59.813 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:33:59 smithi083 ceph-mon[63709]: osdmap e104: 8 total, 7 up, 8 in 2024-09-18T07:33:59.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:59 smithi096 ceph-mon[49483]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:33:59.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:33:59 smithi096 ceph-mon[49483]: osdmap e104: 8 total, 7 up, 8 in 2024-09-18T07:34:00.065 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:33:59 smithi083 podman[72873]: 2024-09-18 07:33:59.8410082 +0000 UTC m=+1.898317111 container cleanup 361ac14ecd8ba6d1f64d60f03c5fd0fc0d88730d505b1b14df167b0780c3c0ae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_CLEAN=True, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, release=754, version=8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, maintainer=Guillaume Abrioux , ceph=True) 2024-09-18T07:34:00.065 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:33:59 smithi083 bash[72873]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2 2024-09-18T07:34:00.317 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:00 smithi083 podman[72890]: 2024-09-18 07:34:00.189069684 +0000 UTC m=+0.939216169 container remove 361ac14ecd8ba6d1f64d60f03c5fd0fc0d88730d505b1b14df167b0780c3c0ae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., RELEASE=HEAD, vcs-type=git, distribution-scope=public, release=754, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870) 2024-09-18T07:34:00.558 INFO:teuthology.orchestra.run.smithi083.stdout: 5 193993 10.23 MB/sec execute 369 sec latency 954.358 ms 2024-09-18T07:34:00.832 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:00 smithi083 ceph-mon[63709]: osdmap e105: 8 total, 7 up, 8 in 2024-09-18T07:34:00.833 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:00 smithi083 ceph-mon[63709]: pgmap v132: 97 pgs: 5 stale+active+clean, 92 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.4 MiB/s wr, 364 op/s 2024-09-18T07:34:00.833 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:00 smithi083 podman[72974]: 2024-09-18 07:34:00.671493562 +0000 UTC m=+0.019431397 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:34:00.833 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:00 smithi083 podman[72974]: 2024-09-18 07:34:00.793014103 +0000 UTC m=+0.140951908 container create 65b93350152b743cd8d8169a976dd375f410b24f09323c616c60172b23204f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-18T07:34:00.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:00 smithi096 ceph-mon[49483]: osdmap e105: 8 total, 7 up, 8 in 2024-09-18T07:34:00.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:00 smithi096 ceph-mon[49483]: pgmap v132: 97 pgs: 5 stale+active+clean, 92 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.4 MiB/s wr, 364 op/s 2024-09-18T07:34:01.083 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:01 smithi083 podman[72974]: 2024-09-18 07:34:01.080293599 +0000 UTC m=+0.428231425 container init 65b93350152b743cd8d8169a976dd375f410b24f09323c616c60172b23204f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-18T07:34:01.334 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:01 smithi083 podman[72974]: 2024-09-18 07:34:01.09275158 +0000 UTC m=+0.440689385 container start 65b93350152b743cd8d8169a976dd375f410b24f09323c616c60172b23204f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:34:01.335 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:01 smithi083 podman[72974]: 2024-09-18 07:34:01.226155983 +0000 UTC m=+0.574093783 container attach 65b93350152b743cd8d8169a976dd375f410b24f09323c616c60172b23204f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-18T07:34:01.559 INFO:teuthology.orchestra.run.smithi083.stdout: 5 194332 10.23 MB/sec execute 370 sec latency 53.197 ms 2024-09-18T07:34:01.589 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:01 smithi083 podman[72974]: 2024-09-18 07:34:01.433301805 +0000 UTC m=+0.781239612 container died 65b93350152b743cd8d8169a976dd375f410b24f09323c616c60172b23204f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True) 2024-09-18T07:34:02.096 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:02.097 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:34:02.348 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:02 smithi083 podman[72997]: 2024-09-18 07:34:02.314667348 +0000 UTC m=+0.868543429 container remove 65b93350152b743cd8d8169a976dd375f410b24f09323c616c60172b23204f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-18T07:34:02.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:02.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:34:02.559 INFO:teuthology.orchestra.run.smithi083.stdout: 5 194662 10.22 MB/sec execute 371 sec latency 74.448 ms 2024-09-18T07:34:02.600 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:02 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-18T07:34:02.600 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:02 smithi083 systemd[1]: Stopped Ceph osd.2 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:34:02.601 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:02 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 30.344s CPU time. 2024-09-18T07:34:02.868 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:02 smithi083 systemd[1]: Starting Ceph osd.2 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:34:03.130 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:02 smithi083 podman[73092]: 2024-09-18 07:34:02.828712048 +0000 UTC m=+0.016641818 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:34:03.130 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:02 smithi083 podman[73092]: 2024-09-18 07:34:02.989607045 +0000 UTC m=+0.177536820 container create 537e7dcd311dc7fb2beef8385b8e5b5b58d8c72784612470cd534c97e3ea3775 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-18T07:34:03.130 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:02 smithi083 ceph-mon[63709]: pgmap v133: 97 pgs: 5 active+undersized, 2 stale+active+clean, 15 active+undersized+degraded, 75 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.4 MiB/s wr, 572 op/s; 123/1554 objects degraded (7.915%) 2024-09-18T07:34:03.130 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:02 smithi083 ceph-mon[63709]: Health check failed: Degraded data redundancy: 123/1554 objects degraded (7.915%), 15 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:03.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:02 smithi096 ceph-mon[49483]: pgmap v133: 97 pgs: 5 active+undersized, 2 stale+active+clean, 15 active+undersized+degraded, 75 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.4 MiB/s wr, 572 op/s; 123/1554 objects degraded (7.915%) 2024-09-18T07:34:03.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:02 smithi096 ceph-mon[49483]: Health check failed: Degraded data redundancy: 123/1554 objects degraded (7.915%), 15 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:03.387 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:03 smithi083 podman[73092]: 2024-09-18 07:34:03.337375959 +0000 UTC m=+0.525305741 container init 537e7dcd311dc7fb2beef8385b8e5b5b58d8c72784612470cd534c97e3ea3775 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:34:03.387 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:03 smithi083 podman[73092]: 2024-09-18 07:34:03.35076541 +0000 UTC m=+0.538695188 container start 537e7dcd311dc7fb2beef8385b8e5b5b58d8c72784612470cd534c97e3ea3775 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2) 2024-09-18T07:34:03.559 INFO:teuthology.orchestra.run.smithi083.stdout: 5 194952 10.21 MB/sec execute 372 sec latency 52.250 ms 2024-09-18T07:34:03.639 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:03 smithi083 podman[73092]: 2024-09-18 07:34:03.468994715 +0000 UTC m=+0.656924504 container attach 537e7dcd311dc7fb2beef8385b8e5b5b58d8c72784612470cd534c97e3ea3775 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-18T07:34:03.639 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:03 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate[73105]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:34:03.639 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:03 smithi083 bash[73092]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:34:03.639 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:03 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate[73105]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:34:03.639 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:03 smithi083 bash[73092]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:34:04.559 INFO:teuthology.orchestra.run.smithi083.stdout: 5 195212 10.21 MB/sec execute 373 sec latency 70.208 ms 2024-09-18T07:34:05.162 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:04 smithi083 ceph-mon[63709]: pgmap v134: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.8 MiB/s wr, 595 op/s; 147/1521 objects degraded (9.665%) 2024-09-18T07:34:05.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:04 smithi096 ceph-mon[49483]: pgmap v134: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.8 MiB/s wr, 595 op/s; 147/1521 objects degraded (9.665%) 2024-09-18T07:34:05.559 INFO:teuthology.orchestra.run.smithi083.stdout: 5 195556 10.22 MB/sec execute 374 sec latency 38.555 ms 2024-09-18T07:34:06.559 INFO:teuthology.orchestra.run.smithi083.stdout: 5 195887 10.22 MB/sec execute 375 sec latency 66.368 ms 2024-09-18T07:34:06.737 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate[73105]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-18T07:34:06.738 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 bash[73092]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-18T07:34:06.738 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate[73105]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-18T07:34:06.738 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 bash[73092]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-18T07:34:06.997 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:06 smithi083 ceph-mon[63709]: pgmap v135: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.0 MiB/s wr, 259 op/s; 147/1521 objects degraded (9.665%) 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate[73105]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 bash[73092]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate[73105]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 bash[73092]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate[73105]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 bash[73092]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate[73105]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 bash[73092]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate[73105]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 bash[73092]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-18T07:34:06.998 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:06 smithi083 podman[73092]: 2024-09-18 07:34:06.971730072 +0000 UTC m=+4.159659841 container died 537e7dcd311dc7fb2beef8385b8e5b5b58d8c72784612470cd534c97e3ea3775 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-18T07:34:07.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:06 smithi096 ceph-mon[49483]: pgmap v135: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.0 MiB/s wr, 259 op/s; 147/1521 objects degraded (9.665%) 2024-09-18T07:34:07.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 196141 10.22 MB/sec execute 376 sec latency 53.856 ms 2024-09-18T07:34:07.758 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:34:08.043 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:07 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 147/1521 objects degraded (9.665%), 20 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:08.044 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:07 smithi083 podman[73260]: 2024-09-18 07:34:07.869614048 +0000 UTC m=+0.891088112 container remove 537e7dcd311dc7fb2beef8385b8e5b5b58d8c72784612470cd534c97e3ea3775 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:34:08.299 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:08 smithi083 podman[73365]: 2024-09-18 07:34:08.012386331 +0000 UTC m=+0.022542914 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:34:08.299 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:08 smithi083 podman[73365]: 2024-09-18 07:34:08.196460095 +0000 UTC m=+0.206616651 container create 1e82d2ea2dc2217717490e81957f23d3e80567c7515bbe9a6c3f0fe7b3ca66c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:34:08.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:07 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 147/1521 objects degraded (9.665%), 20 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:08.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 196457 10.22 MB/sec execute 377 sec latency 39.335 ms 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (5m) 40s ago 14m 18.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (2m) 40s ago 14m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (2m) 2m ago 14m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (5m) 40s ago 14m 44.6M - dad864ee21e9 06ff04a18f50 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (8m) 40s ago 8m 40.6M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (8m) 2m ago 8m 16.0M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (4m) 40s ago 15m 517M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (3m) 2m ago 14m 453M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (3m) 40s ago 16m 48.7M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (3m) 2m ago 14m 35.8M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (8m) 40s ago 8m 270M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (8m) 2m ago 8m 90.1M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:34:08.783 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (5m) 40s ago 14m 10.5M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (4m) 2m ago 13m 12.2M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (99s) 40s ago 13m 248M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (49s) 40s ago 12m 11.8M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (12m) 40s ago 12m 452M 1936M 17.2.0 e1d6a67b021e 361ac14ecd8b 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (11m) 40s ago 11m 579M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (11m) 2m ago 11m 507M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (11m) 2m ago 11m 567M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (10m) 2m ago 10m 502M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (10m) 2m ago 10m 388M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:34:08.784 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (3m) 40s ago 13m 51.0M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:34:08.804 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:08 smithi083 podman[73365]: 2024-09-18 07:34:08.601308951 +0000 UTC m=+0.611465518 container init 1e82d2ea2dc2217717490e81957f23d3e80567c7515bbe9a6c3f0fe7b3ca66c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5) 2024-09-18T07:34:08.804 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:08 smithi083 podman[73365]: 2024-09-18 07:34:08.611507428 +0000 UTC m=+0.621663997 container start 1e82d2ea2dc2217717490e81957f23d3e80567c7515bbe9a6c3f0fe7b3ca66c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:34:08.804 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:08 smithi083 ceph-osd[73406]: -- 172.21.15.83:0/3003606555 <== mon.0 v2:172.21.15.83:3300/0 3 ==== mon_map magic: 0 ==== 357+0+0 (secure 0 0 0) 0x55cada764000 con 0x55cada736000 2024-09-18T07:34:08.804 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:08 smithi083 ceph-osd[73406]: -- 172.21.15.83:0/3003606555 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cada746000 con 0x55cada736000 2024-09-18T07:34:08.804 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:08 smithi083 bash[73365]: 1e82d2ea2dc2217717490e81957f23d3e80567c7515bbe9a6c3f0fe7b3ca66c6 2024-09-18T07:34:08.804 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:08 smithi083 systemd[1]: Started Ceph osd.2 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:34:09.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:08 smithi083 ceph-mon[63709]: pgmap v136: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 462 op/s; 156/1662 objects degraded (9.386%) 2024-09-18T07:34:09.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:08 smithi083 ceph-mon[63709]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:09.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:09.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:09.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:34:09.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:08 smithi096 ceph-mon[49483]: pgmap v136: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 462 op/s; 156/1662 objects degraded (9.386%) 2024-09-18T07:34:09.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:08 smithi096 ceph-mon[49483]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:09.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:09.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:09.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:34:09.539 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:34:09.540 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:34:09.541 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:34:09.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:34:09.541 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:34:09.541 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:34:09.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-18T07:34:09.541 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 6 2024-09-18T07:34:09.541 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:34:09.541 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:34:09.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 196974 10.20 MB/sec execute 378 sec latency 43.866 ms 2024-09-18T07:34:10.059 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:09 smithi083 ceph-mon[63709]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:10.059 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:09 smithi083 ceph-mon[63709]: from='client.44213 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:10.059 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:09 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/2052469125' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:10.121 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:34:10.121 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:34:10.121 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:34:10.121 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:34:10.121 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:34:10.121 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:34:10.121 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:34:10.121 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:34:10.122 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:34:10.122 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "8/23 daemons upgraded", 2024-09-18T07:34:10.122 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:34:10.122 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:34:10.122 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:34:10.272 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:09 smithi096 ceph-mon[49483]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:10.432 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:09 smithi096 ceph-mon[49483]: from='client.44213 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:10.432 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:09 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/2052469125' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:10.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 197584 10.23 MB/sec execute 379 sec latency 70.910 ms 2024-09-18T07:34:10.824 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:10 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[73402]: 2024-09-18T07:34:10.805+0000 7fb9b4b12640 -1 Falling back to public interface 2024-09-18T07:34:10.856 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 156/1662 objects degraded (9.386%), 20 pgs degraded 2024-09-18T07:34:10.856 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-18T07:34:10.856 INFO:teuthology.orchestra.run.smithi083.stdout: osd.2 (root=default,host=smithi083) is down 2024-09-18T07:34:10.856 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 156/1662 objects degraded (9.386%), 20 pgs degraded 2024-09-18T07:34:10.856 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-18T07:34:10.856 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-18T07:34:10.857 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-18T07:34:10.858 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-18T07:34:10.858 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-18T07:34:10.858 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-18T07:34:11.075 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:10 smithi083 ceph-mon[63709]: pgmap v137: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 156/1662 objects degraded (9.386%) 2024-09-18T07:34:11.075 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:10 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/38077388' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:34:11.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:10 smithi096 ceph-mon[49483]: pgmap v137: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 156/1662 objects degraded (9.386%) 2024-09-18T07:34:11.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:10 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/38077388' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:34:11.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 198416 10.24 MB/sec execute 380 sec latency 31.868 ms 2024-09-18T07:34:12.084 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:11 smithi083 ceph-mon[63709]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:12.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:11 smithi096 ceph-mon[49483]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:12.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 198985 10.27 MB/sec execute 381 sec latency 33.544 ms 2024-09-18T07:34:13.093 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:12 smithi083 ceph-mon[63709]: pgmap v138: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.6 MiB/s wr, 625 op/s; 159/1725 objects degraded (9.217%) 2024-09-18T07:34:13.093 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:12 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 159/1725 objects degraded (9.217%), 20 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:13.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:12 smithi096 ceph-mon[49483]: pgmap v138: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.6 MiB/s wr, 625 op/s; 159/1725 objects degraded (9.217%) 2024-09-18T07:34:13.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:12 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 159/1725 objects degraded (9.217%), 20 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:13.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 199475 10.29 MB/sec execute 382 sec latency 42.100 ms 2024-09-18T07:34:14.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 199843 10.30 MB/sec execute 383 sec latency 38.825 ms 2024-09-18T07:34:15.026 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:14 smithi083 ceph-mon[63709]: pgmap v139: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 490 op/s; 157/1698 objects degraded (9.246%) 2024-09-18T07:34:15.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:14 smithi096 ceph-mon[49483]: pgmap v139: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 490 op/s; 157/1698 objects degraded (9.246%) 2024-09-18T07:34:15.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 200383 10.28 MB/sec execute 384 sec latency 30.695 ms 2024-09-18T07:34:16.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 200791 10.29 MB/sec execute 385 sec latency 45.437 ms 2024-09-18T07:34:17.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:16 smithi083 ceph-mon[63709]: pgmap v140: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 474 op/s; 157/1698 objects degraded (9.246%) 2024-09-18T07:34:17.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:17.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:34:17.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:16 smithi096 ceph-mon[49483]: pgmap v140: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 474 op/s; 157/1698 objects degraded (9.246%) 2024-09-18T07:34:17.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:17.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:34:17.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 201356 10.30 MB/sec execute 386 sec latency 44.235 ms 2024-09-18T07:34:18.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:17 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 157/1698 objects degraded (9.246%), 20 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:18.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:17 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:18.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:17 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:18.299 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:18 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[73402]: 2024-09-18T07:34:18.226+0000 7fb9b4b12640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-18T07:34:18.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:17 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 157/1698 objects degraded (9.246%), 20 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:18.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:17 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:18.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:17 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:18.560 INFO:teuthology.orchestra.run.smithi083.stdout: 5 201856 10.30 MB/sec execute 387 sec latency 39.055 ms 2024-09-18T07:34:19.060 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:18 smithi083 ceph-mon[63709]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 786 op/s; 155/1671 objects degraded (9.276%) 2024-09-18T07:34:19.060 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:18 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:19.060 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:18 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:19.311 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:19 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[73402]: 2024-09-18T07:34:19.183+0000 7fb9b4b12640 -1 osd.2 103 log_to_monitors true 2024-09-18T07:34:19.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:18 smithi096 ceph-mon[49483]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 786 op/s; 155/1671 objects degraded (9.276%) 2024-09-18T07:34:19.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:18 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:19.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:18 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:19.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 202225 10.31 MB/sec execute 388 sec latency 38.400 ms 2024-09-18T07:34:20.067 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:19 smithi083 ceph-mon[63709]: from='osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-18T07:34:20.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:19 smithi096 ceph-mon[49483]: from='osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-18T07:34:20.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 202703 10.32 MB/sec execute 389 sec latency 118.126 ms 2024-09-18T07:34:20.582 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:34:20 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[73402]: 2024-09-18T07:34:20.511+0000 7fb9a88b4640 -1 osd.2 103 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:34:21.089 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:20 smithi083 ceph-mon[63709]: pgmap v142: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.0 MiB/s wr, 611 op/s; 155/1671 objects degraded (9.276%) 2024-09-18T07:34:21.089 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:20 smithi083 ceph-mon[63709]: from='osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-18T07:34:21.089 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:20 smithi083 ceph-mon[63709]: osdmap e106: 8 total, 7 up, 8 in 2024-09-18T07:34:21.089 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:20 smithi083 ceph-mon[63709]: from='osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:34:21.089 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:20 smithi083 ceph-mon[63709]: from='osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136]' entity='osd.2' 2024-09-18T07:34:21.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:20 smithi096 ceph-mon[49483]: pgmap v142: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.0 MiB/s wr, 611 op/s; 155/1671 objects degraded (9.276%) 2024-09-18T07:34:21.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:20 smithi096 ceph-mon[49483]: from='osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-18T07:34:21.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:20 smithi096 ceph-mon[49483]: osdmap e106: 8 total, 7 up, 8 in 2024-09-18T07:34:21.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:20 smithi096 ceph-mon[49483]: from='osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:34:21.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:20 smithi096 ceph-mon[49483]: from='osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136]' entity='osd.2' 2024-09-18T07:34:21.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 203112 10.33 MB/sec execute 390 sec latency 79.274 ms 2024-09-18T07:34:22.367 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:21 smithi083 ceph-mon[63709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:34:22.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:21 smithi083 ceph-mon[63709]: osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136] boot 2024-09-18T07:34:22.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:21 smithi083 ceph-mon[63709]: osdmap e107: 8 total, 8 up, 8 in 2024-09-18T07:34:22.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-18T07:34:22.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:21 smithi096 ceph-mon[49483]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:34:22.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:21 smithi096 ceph-mon[49483]: osd.2 [v2:172.21.15.83:6818/867173136,v1:172.21.15.83:6819/867173136] boot 2024-09-18T07:34:22.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:21 smithi096 ceph-mon[49483]: osdmap e107: 8 total, 8 up, 8 in 2024-09-18T07:34:22.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-18T07:34:22.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 203112 10.31 MB/sec execute 391 sec latency 1079.312 ms 2024-09-18T07:34:23.281 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:22 smithi083 ceph-mon[63709]: pgmap v145: 97 pgs: 14 peering, 4 active+undersized, 8 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 9.5 MiB/s wr, 864 op/s; 69/1695 objects degraded (4.071%) 2024-09-18T07:34:23.281 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:22 smithi083 ceph-mon[63709]: osdmap e108: 8 total, 8 up, 8 in 2024-09-18T07:34:23.281 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:22 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 69/1695 objects degraded (4.071%), 8 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:23.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:22 smithi096 ceph-mon[49483]: pgmap v145: 97 pgs: 14 peering, 4 active+undersized, 8 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 9.5 MiB/s wr, 864 op/s; 69/1695 objects degraded (4.071%) 2024-09-18T07:34:23.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:22 smithi096 ceph-mon[49483]: osdmap e108: 8 total, 8 up, 8 in 2024-09-18T07:34:23.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:22 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 69/1695 objects degraded (4.071%), 8 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:23.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 203112 10.28 MB/sec execute 392 sec latency 2079.345 ms 2024-09-18T07:34:24.390 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:24.390 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: osdmap e109: 8 total, 8 up, 8 in 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: pgmap v148: 97 pgs: 14 peering, 4 active+undersized, 8 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 MiB/s wr, 794 op/s; 69/1695 objects degraded (4.071%) 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-18T07:34:24.391 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:24 smithi083 ceph-mon[63709]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-18T07:34:24.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 203283 10.27 MB/sec execute 393 sec latency 2377.486 ms 2024-09-18T07:34:24.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: osdmap e109: 8 total, 8 up, 8 in 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: pgmap v148: 97 pgs: 14 peering, 4 active+undersized, 8 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 MiB/s wr, 794 op/s; 69/1695 objects degraded (4.071%) 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:24.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:24.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-18T07:34:24.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-18T07:34:24.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:24 smithi096 ceph-mon[49483]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-18T07:34:25.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 203474 10.26 MB/sec execute 394 sec latency 51.650 ms 2024-09-18T07:34:25.735 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:25 smithi083 ceph-mon[63709]: osdmap e110: 8 total, 8 up, 8 in 2024-09-18T07:34:25.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:25 smithi096 ceph-mon[49483]: osdmap e110: 8 total, 8 up, 8 in 2024-09-18T07:34:26.508 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:26 smithi083 ceph-mon[63709]: osdmap e111: 8 total, 8 up, 8 in 2024-09-18T07:34:26.509 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:26 smithi083 ceph-mon[63709]: pgmap v151: 97 pgs: 14 peering, 3 active+undersized, 4 active+undersized+degraded, 76 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 12 KiB/s wr, 5 op/s; 42/1695 objects degraded (2.478%); 38 KiB/s, 0 objects/s recovering 2024-09-18T07:34:26.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 203701 10.24 MB/sec execute 395 sec latency 81.836 ms 2024-09-18T07:34:26.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:26 smithi096 ceph-mon[49483]: osdmap e111: 8 total, 8 up, 8 in 2024-09-18T07:34:26.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:26 smithi096 ceph-mon[49483]: pgmap v151: 97 pgs: 14 peering, 3 active+undersized, 4 active+undersized+degraded, 76 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 12 KiB/s wr, 5 op/s; 42/1695 objects degraded (2.478%); 38 KiB/s, 0 objects/s recovering 2024-09-18T07:34:27.540 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:27 smithi083 ceph-mon[63709]: osdmap e112: 8 total, 8 up, 8 in 2024-09-18T07:34:27.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 203939 10.21 MB/sec execute 396 sec latency 69.150 ms 2024-09-18T07:34:27.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:27 smithi096 ceph-mon[49483]: osdmap e112: 8 total, 8 up, 8 in 2024-09-18T07:34:28.559 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:28 smithi083 ceph-mon[63709]: osdmap e113: 8 total, 8 up, 8 in 2024-09-18T07:34:28.560 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:28 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 42/1695 objects degraded (2.478%), 4 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:28.560 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:28 smithi083 ceph-mon[63709]: pgmap v154: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 91 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 2.5 MiB/s wr, 242 op/s; 128/1665 objects degraded (7.688%); 2.3 MiB/s, 30 objects/s recovering 2024-09-18T07:34:28.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 204164 10.20 MB/sec execute 397 sec latency 995.313 ms 2024-09-18T07:34:28.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:28 smithi096 ceph-mon[49483]: osdmap e113: 8 total, 8 up, 8 in 2024-09-18T07:34:28.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:28 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 42/1695 objects degraded (2.478%), 4 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:28.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:28 smithi096 ceph-mon[49483]: pgmap v154: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 91 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 2.5 MiB/s wr, 242 op/s; 128/1665 objects degraded (7.688%); 2.3 MiB/s, 30 objects/s recovering 2024-09-18T07:34:29.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 204415 10.19 MB/sec execute 398 sec latency 560.798 ms 2024-09-18T07:34:29.587 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:29 smithi083 ceph-mon[63709]: osdmap e114: 8 total, 8 up, 8 in 2024-09-18T07:34:29.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:29 smithi096 ceph-mon[49483]: osdmap e114: 8 total, 8 up, 8 in 2024-09-18T07:34:30.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 204805 10.20 MB/sec execute 399 sec latency 62.525 ms 2024-09-18T07:34:30.616 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:30 smithi083 ceph-mon[63709]: osdmap e115: 8 total, 8 up, 8 in 2024-09-18T07:34:30.616 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:30 smithi083 ceph-mon[63709]: pgmap v157: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 91 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.5 MiB/s wr, 236 op/s; 128/1665 objects degraded (7.688%); 2.2 MiB/s, 30 objects/s recovering 2024-09-18T07:34:30.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:30 smithi096 ceph-mon[49483]: osdmap e115: 8 total, 8 up, 8 in 2024-09-18T07:34:30.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:30 smithi096 ceph-mon[49483]: pgmap v157: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 91 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.5 MiB/s wr, 236 op/s; 128/1665 objects degraded (7.688%); 2.2 MiB/s, 30 objects/s recovering 2024-09-18T07:34:31.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 205168 10.19 MB/sec execute 400 sec latency 939.037 ms 2024-09-18T07:34:31.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:31 smithi083 ceph-mon[63709]: osdmap e116: 8 total, 8 up, 8 in 2024-09-18T07:34:31.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:31.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:34:31.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:31 smithi096 ceph-mon[49483]: osdmap e116: 8 total, 8 up, 8 in 2024-09-18T07:34:31.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:31.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:34:32.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 205577 10.19 MB/sec execute 401 sec latency 754.161 ms 2024-09-18T07:34:32.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:32 smithi083 ceph-mon[63709]: osdmap e117: 8 total, 8 up, 8 in 2024-09-18T07:34:32.650 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:32 smithi083 ceph-mon[63709]: pgmap v160: 97 pgs: 1 active+undersized+remapped, 1 activating, 1 peering, 94 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.2 MiB/s wr, 519 op/s; 2.0 KiB/s, 32 objects/s recovering 2024-09-18T07:34:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:32 smithi096 ceph-mon[49483]: osdmap e117: 8 total, 8 up, 8 in 2024-09-18T07:34:32.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:32 smithi096 ceph-mon[49483]: pgmap v160: 97 pgs: 1 active+undersized+remapped, 1 activating, 1 peering, 94 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.2 MiB/s wr, 519 op/s; 2.0 KiB/s, 32 objects/s recovering 2024-09-18T07:34:33.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 206015 10.21 MB/sec execute 402 sec latency 353.983 ms 2024-09-18T07:34:33.659 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:33 smithi083 ceph-mon[63709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 128/1665 objects degraded (7.688%), 5 pgs degraded) 2024-09-18T07:34:33.660 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:33 smithi083 ceph-mon[63709]: Cluster is now healthy 2024-09-18T07:34:33.660 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:33 smithi083 ceph-mon[63709]: osdmap e118: 8 total, 8 up, 8 in 2024-09-18T07:34:33.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:33 smithi096 ceph-mon[49483]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 128/1665 objects degraded (7.688%), 5 pgs degraded) 2024-09-18T07:34:33.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:33 smithi096 ceph-mon[49483]: Cluster is now healthy 2024-09-18T07:34:33.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:33 smithi096 ceph-mon[49483]: osdmap e118: 8 total, 8 up, 8 in 2024-09-18T07:34:34.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 206480 10.23 MB/sec execute 403 sec latency 40.051 ms 2024-09-18T07:34:34.667 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:34 smithi083 ceph-mon[63709]: pgmap v162: 97 pgs: 1 active+undersized+remapped, 1 activating, 1 peering, 94 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.1 MiB/s wr, 512 op/s; 2.0 KiB/s, 31 objects/s recovering 2024-09-18T07:34:34.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:34 smithi096 ceph-mon[49483]: pgmap v162: 97 pgs: 1 active+undersized+remapped, 1 activating, 1 peering, 94 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.1 MiB/s wr, 512 op/s; 2.0 KiB/s, 31 objects/s recovering 2024-09-18T07:34:35.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 206839 10.23 MB/sec execute 404 sec latency 33.971 ms 2024-09-18T07:34:36.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 207147 10.21 MB/sec execute 405 sec latency 52.964 ms 2024-09-18T07:34:36.704 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:36 smithi083 ceph-mon[63709]: pgmap v163: 97 pgs: 1 active+undersized+remapped, 1 activating, 1 peering, 94 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.8 MiB/s wr, 349 op/s; 1.3 KiB/s, 21 objects/s recovering 2024-09-18T07:34:37.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:36 smithi096 ceph-mon[49483]: pgmap v163: 97 pgs: 1 active+undersized+remapped, 1 activating, 1 peering, 94 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.8 MiB/s wr, 349 op/s; 1.3 KiB/s, 21 objects/s recovering 2024-09-18T07:34:37.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 207532 10.20 MB/sec execute 406 sec latency 100.097 ms 2024-09-18T07:34:38.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 207941 10.20 MB/sec execute 407 sec latency 48.200 ms 2024-09-18T07:34:38.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:38 smithi083 ceph-mon[63709]: pgmap v164: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 7.3 MiB/s wr, 721 op/s; 1.1 KiB/s, 18 objects/s recovering 2024-09-18T07:34:39.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:38 smithi096 ceph-mon[49483]: pgmap v164: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 7.3 MiB/s wr, 721 op/s; 1.1 KiB/s, 18 objects/s recovering 2024-09-18T07:34:39.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 208339 10.19 MB/sec execute 408 sec latency 48.557 ms 2024-09-18T07:34:39.786 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:39 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-18T07:34:39.786 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:39 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-18T07:34:39.786 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:39 smithi083 ceph-mon[63709]: Upgrade: osd.3 is safe to restart 2024-09-18T07:34:40.022 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:39 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-18T07:34:40.022 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:39 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-18T07:34:40.023 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:39 smithi096 ceph-mon[49483]: Upgrade: osd.3 is safe to restart 2024-09-18T07:34:40.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 208885 10.20 MB/sec execute 409 sec latency 45.160 ms 2024-09-18T07:34:40.796 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:40 smithi083 ceph-mon[63709]: pgmap v165: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 5.1 MiB/s wr, 458 op/s; 0 B/s, 9 objects/s recovering 2024-09-18T07:34:41.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:40 smithi096 ceph-mon[49483]: pgmap v165: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 5.1 MiB/s wr, 458 op/s; 0 B/s, 9 objects/s recovering 2024-09-18T07:34:41.347 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:34:41.562 INFO:teuthology.orchestra.run.smithi083.stdout: 5 209192 10.21 MB/sec execute 410 sec latency 46.445 ms 2024-09-18T07:34:42.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:41 smithi083 ceph-mon[63709]: Upgrade: Updating osd.3 2024-09-18T07:34:42.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:41 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:42.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:41 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-18T07:34:42.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:41 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:34:42.066 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:41 smithi083 ceph-mon[63709]: Deploying daemon osd.3 on smithi083 2024-09-18T07:34:42.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:41 smithi096 ceph-mon[49483]: Upgrade: Updating osd.3 2024-09-18T07:34:42.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:41 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:42.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:41 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-18T07:34:42.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:41 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:34:42.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:41 smithi096 ceph-mon[49483]: Deploying daemon osd.3 on smithi083 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (6m) 24s ago 15m 20.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (2m) 24s ago 15m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (2m) 2m ago 14m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (5m) 24s ago 15m 46.0M - dad864ee21e9 06ff04a18f50 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (9m) 24s ago 9m 40.4M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (9m) 2m ago 9m 16.0M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (5m) 24s ago 16m 518M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:34:42.269 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (4m) 24s ago 16m 50.6M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (3m) 2m ago 14m 35.8M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (9m) 24s ago 9m 285M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (8m) 2m ago 8m 90.1M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (5m) 24s ago 15m 9.78M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (5m) 2m ago 14m 12.2M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2m) 24s ago 13m 310M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (82s) 24s ago 13m 236M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (34s) 24s ago 12m 12.2M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (12m) 24s ago 12m 355M 1936M 17.2.0 e1d6a67b021e 764c8007ebe5 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (12m) 2m ago 12m 507M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (11m) 2m ago 11m 567M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (11m) 2m ago 11m 502M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:34:42.270 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (10m) 2m ago 10m 388M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:34:42.271 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (4m) 24s ago 14m 51.1M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:34:42.561 INFO:teuthology.orchestra.run.smithi083.stdout: 5 209485 10.21 MB/sec execute 411 sec latency 53.409 ms 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 3 2024-09-18T07:34:42.853 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:34:42.854 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:34:42.854 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:34:42.854 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:34:42.854 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:34:42.854 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-18T07:34:42.854 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 7 2024-09-18T07:34:42.854 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:34:42.854 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:34:43.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:42 smithi083 ceph-mon[63709]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:43.079 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:42 smithi083 ceph-mon[63709]: pgmap v166: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 603 op/s 2024-09-18T07:34:43.080 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:42 smithi083 ceph-mon[63709]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:43.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:42 smithi096 ceph-mon[49483]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:43.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:42 smithi096 ceph-mon[49483]: pgmap v166: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.0 MiB/s wr, 603 op/s 2024-09-18T07:34:43.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:42 smithi096 ceph-mon[49483]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:43.349 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:34:43.349 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:34:43.349 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:34:43.349 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:34:43.349 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:34:43.349 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:34:43.349 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:34:43.349 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:34:43.349 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:34:43.350 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "9/23 daemons upgraded", 2024-09-18T07:34:43.350 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:34:43.350 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:34:43.350 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:34:43.562 INFO:teuthology.orchestra.run.smithi083.stdout: 5 209854 10.22 MB/sec execute 412 sec latency 44.149 ms 2024-09-18T07:34:43.918 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:34:44.099 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:43 smithi083 ceph-mon[63709]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:44.100 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:43 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/2132173209' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:44.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:43 smithi096 ceph-mon[49483]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:44.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:43 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/2132173209' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:34:44.562 INFO:teuthology.orchestra.run.smithi083.stdout: 5 210182 10.22 MB/sec execute 413 sec latency 56.537 ms 2024-09-18T07:34:45.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:44 smithi083 ceph-mon[63709]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:45.113 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:44 smithi083 ceph-mon[63709]: pgmap v167: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 547 op/s 2024-09-18T07:34:45.113 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:44 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/893167128' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:34:45.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:44 smithi096 ceph-mon[49483]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:34:45.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:44 smithi096 ceph-mon[49483]: pgmap v167: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 547 op/s 2024-09-18T07:34:45.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:44 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/893167128' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:34:45.562 INFO:teuthology.orchestra.run.smithi083.stdout: 5 210432 10.21 MB/sec execute 414 sec latency 37.630 ms 2024-09-18T07:34:45.618 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:45 smithi083 systemd[1]: Stopping Ceph osd.3 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:34:46.133 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:45 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[45110]: 2024-09-18T07:34:45.908+0000 7f7f16c3a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:34:46.133 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:45 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[45110]: 2024-09-18T07:34:45.908+0000 7f7f16c3a700 -1 osd.3 118 *** Got signal Terminated *** 2024-09-18T07:34:46.133 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:45 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[45110]: 2024-09-18T07:34:45.908+0000 7f7f16c3a700 -1 osd.3 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:34:46.562 INFO:teuthology.orchestra.run.smithi083.stdout: 5 210699 10.20 MB/sec execute 415 sec latency 69.723 ms 2024-09-18T07:34:46.894 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:46 smithi083 ceph-mon[63709]: pgmap v168: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 504 op/s 2024-09-18T07:34:46.894 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:46 smithi083 ceph-mon[63709]: osd.3 marked itself down and dead 2024-09-18T07:34:46.894 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:46.894 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:34:46.894 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:47.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:46 smithi096 ceph-mon[49483]: pgmap v168: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 504 op/s 2024-09-18T07:34:47.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:46 smithi096 ceph-mon[49483]: osd.3 marked itself down and dead 2024-09-18T07:34:47.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:47.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:34:47.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:34:47.562 INFO:teuthology.orchestra.run.smithi083.stdout: 5 210794 10.18 MB/sec execute 416 sec latency 702.296 ms 2024-09-18T07:34:47.849 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:47 smithi083 podman[75368]: 2024-09-18 07:34:47.539102146 +0000 UTC m=+1.774953391 container died 764c8007ebe541373e14aa529b91ed670e06984849fc59a83e3cd2931d90bd93 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, distribution-scope=public, io.openshift.expose-services=, GIT_CLEAN=True, ceph=True, vcs-type=git, com.redhat.component=centos-stream-container, architecture=x86_64, vendor=Red Hat, Inc., release=754, GIT_BRANCH=HEAD) 2024-09-18T07:34:48.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:47 smithi083 ceph-mon[63709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:34:48.101 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:47 smithi083 ceph-mon[63709]: osdmap e119: 8 total, 7 up, 8 in 2024-09-18T07:34:48.101 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:48 smithi083 podman[75368]: 2024-09-18 07:34:48.054801655 +0000 UTC m=+2.290652896 container cleanup 764c8007ebe541373e14aa529b91ed670e06984849fc59a83e3cd2931d90bd93 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3, distribution-scope=public, version=8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , name=centos-stream, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.k8s.display-name=CentOS Stream 8, vcs-type=git) 2024-09-18T07:34:48.101 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:48 smithi083 bash[75368]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3 2024-09-18T07:34:48.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:47 smithi096 ceph-mon[49483]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:34:48.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:47 smithi096 ceph-mon[49483]: osdmap e119: 8 total, 7 up, 8 in 2024-09-18T07:34:48.562 INFO:teuthology.orchestra.run.smithi083.stdout: 5 211015 10.16 MB/sec execute 417 sec latency 1052.594 ms 2024-09-18T07:34:48.606 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:48 smithi083 podman[75384]: 2024-09-18 07:34:48.408724975 +0000 UTC m=+0.865195474 container remove 764c8007ebe541373e14aa529b91ed670e06984849fc59a83e3cd2931d90bd93 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, maintainer=Guillaume Abrioux , release=754, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, vendor=Red Hat, Inc., ceph=True, distribution-scope=public, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, name=centos-stream) 2024-09-18T07:34:49.107 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:48 smithi083 ceph-mon[63709]: pgmap v170: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 501 op/s 2024-09-18T07:34:49.107 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:48 smithi083 ceph-mon[63709]: osdmap e120: 8 total, 7 up, 8 in 2024-09-18T07:34:49.108 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:48 smithi083 podman[75472]: 2024-09-18 07:34:48.844296996 +0000 UTC m=+0.020466381 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:34:49.108 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:49 smithi083 podman[75472]: 2024-09-18 07:34:49.054847592 +0000 UTC m=+0.231016966 container create a64b1270c5d831b775ec198d2102004e67fda397571d4a792f7bcfa0cc61ad6e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-18T07:34:49.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:48 smithi096 ceph-mon[49483]: pgmap v170: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 501 op/s 2024-09-18T07:34:49.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:48 smithi096 ceph-mon[49483]: osdmap e120: 8 total, 7 up, 8 in 2024-09-18T07:34:49.562 INFO:teuthology.orchestra.run.smithi083.stdout: 5 211380 10.15 MB/sec execute 418 sec latency 82.917 ms 2024-09-18T07:34:49.625 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:49 smithi083 podman[75472]: 2024-09-18 07:34:49.562916412 +0000 UTC m=+0.739085802 container init a64b1270c5d831b775ec198d2102004e67fda397571d4a792f7bcfa0cc61ad6e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:34:49.625 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:49 smithi083 podman[75472]: 2024-09-18 07:34:49.573491804 +0000 UTC m=+0.749661188 container start a64b1270c5d831b775ec198d2102004e67fda397571d4a792f7bcfa0cc61ad6e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True) 2024-09-18T07:34:49.881 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:49 smithi083 podman[75472]: 2024-09-18 07:34:49.70647519 +0000 UTC m=+0.882644572 container attach a64b1270c5d831b775ec198d2102004e67fda397571d4a792f7bcfa0cc61ad6e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-18T07:34:50.131 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:49 smithi083 podman[75472]: 2024-09-18 07:34:49.884346959 +0000 UTC m=+1.060516342 container died a64b1270c5d831b775ec198d2102004e67fda397571d4a792f7bcfa0cc61ad6e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-18T07:34:50.562 INFO:teuthology.orchestra.run.smithi083.stdout: 5 211708 10.15 MB/sec execute 419 sec latency 59.618 ms 2024-09-18T07:34:50.892 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:50 smithi083 ceph-mon[63709]: pgmap v172: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.2 MiB/s wr, 250 op/s 2024-09-18T07:34:51.144 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:50 smithi083 podman[75495]: 2024-09-18 07:34:50.998690785 +0000 UTC m=+1.098045753 container remove a64b1270c5d831b775ec198d2102004e67fda397571d4a792f7bcfa0cc61ad6e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-18T07:34:51.144 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:51 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-18T07:34:51.144 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:51 smithi083 systemd[1]: Stopped Ceph osd.3 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:34:51.144 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:51 smithi083 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.3.service: Consumed 4min 4.930s CPU time. 2024-09-18T07:34:51.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:50 smithi096 ceph-mon[49483]: pgmap v172: 97 pgs: 16 stale+active+clean, 81 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.2 MiB/s wr, 250 op/s 2024-09-18T07:34:51.566 INFO:teuthology.orchestra.run.smithi083.stdout: 5 212268 10.16 MB/sec execute 420 sec latency 32.734 ms 2024-09-18T07:34:51.662 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:51 smithi083 systemd[1]: Starting Ceph osd.3 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:34:51.918 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:51 smithi083 podman[75590]: 2024-09-18 07:34:51.586551472 +0000 UTC m=+0.024153831 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:34:51.918 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:51 smithi083 podman[75590]: 2024-09-18 07:34:51.759213651 +0000 UTC m=+0.196815991 container create a1fd2bc84513b6ab7439aaa1fd9acf6229da93c72ae1a06393cc0a1647cf3113 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5) 2024-09-18T07:34:52.175 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:51 smithi083 ceph-mon[63709]: Health check failed: Degraded data redundancy: 202/1575 objects degraded (12.825%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:52.235 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:52 smithi083 podman[75590]: 2024-09-18 07:34:52.226904979 +0000 UTC m=+0.664507331 container init a1fd2bc84513b6ab7439aaa1fd9acf6229da93c72ae1a06393cc0a1647cf3113 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-18T07:34:52.251 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:52 smithi083 podman[75590]: 2024-09-18 07:34:52.244213003 +0000 UTC m=+0.681815352 container start a1fd2bc84513b6ab7439aaa1fd9acf6229da93c72ae1a06393cc0a1647cf3113 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:34:52.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:51 smithi096 ceph-mon[49483]: Health check failed: Degraded data redundancy: 202/1575 objects degraded (12.825%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:52.426 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:52 smithi083 podman[75590]: 2024-09-18 07:34:52.415015802 +0000 UTC m=+0.852618150 container attach a1fd2bc84513b6ab7439aaa1fd9acf6229da93c72ae1a06393cc0a1647cf3113 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-18T07:34:52.427 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:52 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate[75604]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:34:52.427 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:52 smithi083 bash[75590]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:34:52.440 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:52 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate[75604]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:34:52.440 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:52 smithi083 bash[75590]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:34:52.572 INFO:teuthology.orchestra.run.smithi083.stdout: 5 212765 10.17 MB/sec execute 421 sec latency 34.722 ms 2024-09-18T07:34:52.948 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:52 smithi083 ceph-mon[63709]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 571 op/s; 202/1575 objects degraded (12.825%) 2024-09-18T07:34:53.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:52 smithi096 ceph-mon[49483]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 571 op/s; 202/1575 objects degraded (12.825%) 2024-09-18T07:34:53.572 INFO:teuthology.orchestra.run.smithi083.stdout: 5 213213 10.18 MB/sec execute 422 sec latency 40.938 ms 2024-09-18T07:34:54.572 INFO:teuthology.orchestra.run.smithi083.stdout: 5 213839 10.22 MB/sec execute 423 sec latency 29.848 ms 2024-09-18T07:34:54.984 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:54 smithi083 ceph-mon[63709]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.7 MiB/s wr, 568 op/s; 202/1575 objects degraded (12.825%) 2024-09-18T07:34:55.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:54 smithi096 ceph-mon[49483]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.7 MiB/s wr, 568 op/s; 202/1575 objects degraded (12.825%) 2024-09-18T07:34:55.488 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate[75604]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-18T07:34:55.489 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 bash[75590]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-18T07:34:55.489 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate[75604]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-18T07:34:55.489 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 bash[75590]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-18T07:34:55.582 INFO:teuthology.orchestra.run.smithi083.stdout: 5 214274 10.21 MB/sec execute 424 sec latency 33.778 ms 2024-09-18T07:34:55.994 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate[75604]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-18T07:34:55.994 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 bash[75590]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-18T07:34:55.994 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate[75604]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-18T07:34:55.994 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 bash[75590]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-18T07:34:55.994 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate[75604]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-18T07:34:55.994 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 bash[75590]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-18T07:34:55.994 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate[75604]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-18T07:34:55.994 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 bash[75590]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-18T07:34:55.994 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate[75604]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-18T07:34:55.995 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 bash[75590]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-18T07:34:55.995 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:55 smithi083 podman[75773]: 2024-09-18 07:34:55.93052426 +0000 UTC m=+0.024785477 container died a1fd2bc84513b6ab7439aaa1fd9acf6229da93c72ae1a06393cc0a1647cf3113 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:34:56.582 INFO:teuthology.orchestra.run.smithi083.stdout: 5 214615 10.20 MB/sec execute 425 sec latency 73.720 ms 2024-09-18T07:34:57.037 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:56 smithi083 ceph-mon[63709]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 2.3 MiB/s wr, 297 op/s; 202/1572 objects degraded (12.850%) 2024-09-18T07:34:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:56 smithi096 ceph-mon[49483]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 2.3 MiB/s wr, 297 op/s; 202/1572 objects degraded (12.850%) 2024-09-18T07:34:57.582 INFO:teuthology.orchestra.run.smithi083.stdout: 5 214983 10.19 MB/sec execute 426 sec latency 46.016 ms 2024-09-18T07:34:57.796 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:57 smithi083 podman[75773]: 2024-09-18 07:34:57.777178849 +0000 UTC m=+1.871440052 container remove a1fd2bc84513b6ab7439aaa1fd9acf6229da93c72ae1a06393cc0a1647cf3113 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-18T07:34:58.047 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:58 smithi083 podman[75845]: 2024-09-18 07:34:57.936919196 +0000 UTC m=+0.023590330 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:34:58.048 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:57 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 202/1572 objects degraded (12.850%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:58.299 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:58 smithi083 podman[75845]: 2024-09-18 07:34:58.185562344 +0000 UTC m=+0.272233476 container create 9e0abb53ec3ce72e0ecb373c438a4f7b9993bf2fa4992647943320c0d86aab44 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:34:58.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:57 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 202/1572 objects degraded (12.850%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:34:58.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 215432 10.19 MB/sec execute 427 sec latency 41.163 ms 2024-09-18T07:34:58.810 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:58 smithi083 podman[75845]: 2024-09-18 07:34:58.674015058 +0000 UTC m=+0.760686190 container init 9e0abb53ec3ce72e0ecb373c438a4f7b9993bf2fa4992647943320c0d86aab44 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True) 2024-09-18T07:34:58.811 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:58 smithi083 podman[75845]: 2024-09-18 07:34:58.685152748 +0000 UTC m=+0.771823893 container start 9e0abb53ec3ce72e0ecb373c438a4f7b9993bf2fa4992647943320c0d86aab44 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-18T07:34:58.811 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:58 smithi083 ceph-osd[75862]: -- 172.21.15.83:0/3986961321 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5648d26c61e0 con 0x5648d2652000 2024-09-18T07:34:59.065 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:58 smithi083 bash[75845]: 9e0abb53ec3ce72e0ecb373c438a4f7b9993bf2fa4992647943320c0d86aab44 2024-09-18T07:34:59.065 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:34:58 smithi083 systemd[1]: Started Ceph osd.3 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:34:59.065 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:58 smithi083 ceph-mon[63709]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.8 MiB/s wr, 599 op/s; 221/1686 objects degraded (13.108%) 2024-09-18T07:34:59.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:58 smithi096 ceph-mon[49483]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.8 MiB/s wr, 599 op/s; 221/1686 objects degraded (13.108%) 2024-09-18T07:34:59.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 215822 10.19 MB/sec execute 428 sec latency 51.312 ms 2024-09-18T07:35:00.075 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:59 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:00.075 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:59 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:00.075 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:34:59 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:35:00.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:59 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:00.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:59 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:00.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:34:59 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:35:00.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 216301 10.19 MB/sec execute 429 sec latency 32.574 ms 2024-09-18T07:35:01.086 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:35:01 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[75858]: 2024-09-18T07:35:01.044+0000 7f9de4524640 -1 Falling back to public interface 2024-09-18T07:35:01.087 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:00 smithi083 ceph-mon[63709]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 513 op/s; 221/1686 objects degraded (13.108%) 2024-09-18T07:35:01.087 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:00 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:01.087 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:00 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:35:01.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:00 smithi096 ceph-mon[49483]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 513 op/s; 221/1686 objects degraded (13.108%) 2024-09-18T07:35:01.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:00 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:01.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:00 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:35:01.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 216776 10.22 MB/sec execute 430 sec latency 30.027 ms 2024-09-18T07:35:02.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 217212 10.23 MB/sec execute 431 sec latency 36.980 ms 2024-09-18T07:35:03.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:02 smithi083 ceph-mon[63709]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 750 op/s; 206/1617 objects degraded (12.740%) 2024-09-18T07:35:03.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:02 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 206/1617 objects degraded (12.740%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:03.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:02 smithi096 ceph-mon[49483]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 750 op/s; 206/1617 objects degraded (12.740%) 2024-09-18T07:35:03.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:02 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 206/1617 objects degraded (12.740%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:03.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 217555 10.23 MB/sec execute 432 sec latency 45.697 ms 2024-09-18T07:35:04.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 217770 10.21 MB/sec execute 433 sec latency 66.694 ms 2024-09-18T07:35:05.182 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:04 smithi083 ceph-mon[63709]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 536 op/s; 206/1617 objects degraded (12.740%) 2024-09-18T07:35:05.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:04 smithi096 ceph-mon[49483]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 536 op/s; 206/1617 objects degraded (12.740%) 2024-09-18T07:35:05.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 217968 10.20 MB/sec execute 434 sec latency 68.066 ms 2024-09-18T07:35:06.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 218186 10.18 MB/sec execute 435 sec latency 89.386 ms 2024-09-18T07:35:07.012 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:06 smithi083 ceph-mon[63709]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 206/1617 objects degraded (12.740%) 2024-09-18T07:35:07.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:06 smithi096 ceph-mon[49483]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 206/1617 objects degraded (12.740%) 2024-09-18T07:35:07.586 INFO:teuthology.orchestra.run.smithi083.stdout: 5 218449 10.17 MB/sec execute 436 sec latency 51.397 ms 2024-09-18T07:35:08.040 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:08 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 198/1554 objects degraded (12.741%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:08.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:08 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 198/1554 objects degraded (12.741%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:08.542 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:35:08 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[75858]: 2024-09-18T07:35:08.455+0000 7f9de4524640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-18T07:35:08.594 INFO:teuthology.orchestra.run.smithi083.stdout: 5 218837 10.16 MB/sec execute 437 sec latency 77.620 ms 2024-09-18T07:35:09.045 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:09 smithi083 ceph-mon[63709]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 723 op/s; 198/1554 objects degraded (12.741%) 2024-09-18T07:35:09.046 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:09 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:09.046 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:09 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:09.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:09 smithi096 ceph-mon[49483]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 723 op/s; 198/1554 objects degraded (12.741%) 2024-09-18T07:35:09.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:09 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:09.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:09 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:09.594 INFO:teuthology.orchestra.run.smithi083.stdout: 5 219237 10.17 MB/sec execute 438 sec latency 41.400 ms 2024-09-18T07:35:10.063 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:35:09 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[75858]: 2024-09-18T07:35:09.941+0000 7f9de4524640 -1 osd.3 118 log_to_monitors true 2024-09-18T07:35:10.594 INFO:teuthology.orchestra.run.smithi083.stdout: 5 219706 10.17 MB/sec execute 439 sec latency 43.501 ms 2024-09-18T07:35:10.835 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:10 smithi083 ceph-mon[63709]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 438 op/s; 198/1554 objects degraded (12.741%) 2024-09-18T07:35:10.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:10 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:10.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:10 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:10.836 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:10 smithi083 ceph-mon[63709]: from='osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-18T07:35:11.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:10 smithi096 ceph-mon[49483]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 438 op/s; 198/1554 objects degraded (12.741%) 2024-09-18T07:35:11.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:10 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:11.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:10 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:11.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:10 smithi096 ceph-mon[49483]: from='osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-18T07:35:11.378 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:35:11 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[75858]: 2024-09-18T07:35:11.294+0000 7f9dd82c6640 -1 osd.3 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:35:11.594 INFO:teuthology.orchestra.run.smithi083.stdout: 5 219976 10.17 MB/sec execute 440 sec latency 108.037 ms 2024-09-18T07:35:11.881 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:11 smithi083 ceph-mon[63709]: from='osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-18T07:35:11.881 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:11 smithi083 ceph-mon[63709]: osdmap e121: 8 total, 7 up, 8 in 2024-09-18T07:35:11.881 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:11 smithi083 ceph-mon[63709]: from='osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:35:11.881 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:11 smithi083 ceph-mon[63709]: from='osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809]' entity='osd.3' 2024-09-18T07:35:12.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:11 smithi096 ceph-mon[49483]: from='osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-18T07:35:12.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:11 smithi096 ceph-mon[49483]: osdmap e121: 8 total, 7 up, 8 in 2024-09-18T07:35:12.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:11 smithi096 ceph-mon[49483]: from='osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]} : dispatch 2024-09-18T07:35:12.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:11 smithi096 ceph-mon[49483]: from='osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809]' entity='osd.3' 2024-09-18T07:35:12.594 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220333 10.18 MB/sec execute 441 sec latency 278.435 ms 2024-09-18T07:35:12.912 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:12 smithi083 ceph-mon[63709]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 464 op/s; 205/1578 objects degraded (12.991%) 2024-09-18T07:35:12.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:12 smithi083 ceph-mon[63709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:35:12.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:12 smithi083 ceph-mon[63709]: osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809] boot 2024-09-18T07:35:12.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:12 smithi083 ceph-mon[63709]: osdmap e122: 8 total, 8 up, 8 in 2024-09-18T07:35:12.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-18T07:35:12.913 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:12 smithi083 ceph-mon[63709]: osdmap e123: 8 total, 8 up, 8 in 2024-09-18T07:35:13.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:12 smithi096 ceph-mon[49483]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 464 op/s; 205/1578 objects degraded (12.991%) 2024-09-18T07:35:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:12 smithi096 ceph-mon[49483]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:35:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:12 smithi096 ceph-mon[49483]: osd.3 [v2:172.21.15.83:6826/3100225809,v1:172.21.15.83:6827/3100225809] boot 2024-09-18T07:35:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:12 smithi096 ceph-mon[49483]: osdmap e122: 8 total, 8 up, 8 in 2024-09-18T07:35:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-18T07:35:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:12 smithi096 ceph-mon[49483]: osdmap e123: 8 total, 8 up, 8 in 2024-09-18T07:35:13.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220492 10.17 MB/sec execute 442 sec latency 1285.231 ms 2024-09-18T07:35:14.517 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:35:14.593 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:14 smithi083 ceph-mon[63709]: osdmap e124: 8 total, 8 up, 8 in 2024-09-18T07:35:14.594 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:14 smithi083 ceph-mon[63709]: pgmap v188: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.4 MiB/s wr, 596 op/s; 205/1578 objects degraded (12.991%) 2024-09-18T07:35:14.600 INFO:teuthology.orchestra.run.smithi083.stdout: 5 220880 10.17 MB/sec execute 443 sec latency 1520.458 ms 2024-09-18T07:35:14.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:14 smithi096 ceph-mon[49483]: osdmap e124: 8 total, 8 up, 8 in 2024-09-18T07:35:14.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:14 smithi096 ceph-mon[49483]: pgmap v188: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.4 MiB/s wr, 596 op/s; 205/1578 objects degraded (12.991%) 2024-09-18T07:35:15.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 221255 10.18 MB/sec execute 444 sec latency 37.770 ms 2024-09-18T07:35:15.607 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:15 smithi083 ceph-mon[63709]: from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:15.608 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:15 smithi083 ceph-mon[63709]: osdmap e125: 8 total, 8 up, 8 in 2024-09-18T07:35:15.608 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:15 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:15.608 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:15 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:15.608 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:15 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:35:15.608 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:15 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:35:15.608 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:15 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:15.608 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:15 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (6m) 6s ago 15m 20.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (3m) 6s ago 15m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (3m) 3m ago 15m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (6m) 6s ago 15m 45.0M - dad864ee21e9 06ff04a18f50 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (9m) 6s ago 9m 41.2M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (9m) 3m ago 9m 16.0M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (5m) 6s ago 17m 520M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (4m) 3m ago 15m 453M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (4m) 6s ago 17m 51.7M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (4m) 3m ago 15m 35.8M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:35:15.655 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (9m) 6s ago 9m 279M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (9m) 3m ago 9m 90.1M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (6m) 6s ago 15m 9.79M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (5m) 3m ago 15m 12.2M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (2m) 6s ago 14m 365M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (116s) 6s ago 13m 277M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (67s) 6s ago 13m 212M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (17s) 6s ago 13m 11.7M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (12m) 3m ago 12m 507M 2704M 17.2.0 e1d6a67b021e 305fa8f943da 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (12m) 3m ago 12m 567M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (11m) 3m ago 11m 502M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (11m) 3m ago 11m 388M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:35:15.656 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (5m) 6s ago 14m 53.3M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:35:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:15 smithi096 ceph-mon[49483]: from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:15 smithi096 ceph-mon[49483]: osdmap e125: 8 total, 8 up, 8 in 2024-09-18T07:35:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:15 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:15 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:15 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:35:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:15 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:35:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:15 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:15.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:15 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:16.353 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:35:16.353 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:35:16.353 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 4 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 8 2024-09-18T07:35:16.354 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:35:16.355 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:35:16.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 221499 10.16 MB/sec execute 445 sec latency 994.040 ms 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='client.34330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: pgmap v191: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized, 24 active+undersized+degraded, 59 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 23 KiB/s wr, 7 op/s; 201/1578 objects degraded (12.738%); 49 KiB/s, 3 objects/s recovering 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: osdmap e126: 8 total, 8 up, 8 in 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:16.627 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:16.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-18T07:35:16.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-18T07:35:16.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-18T07:35:16.628 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:16 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/3958285469' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='client.34330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: pgmap v191: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized, 24 active+undersized+degraded, 59 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 23 KiB/s wr, 7 op/s; 201/1578 objects degraded (12.738%); 49 KiB/s, 3 objects/s recovering 2024-09-18T07:35:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: osdmap e126: 8 total, 8 up, 8 in 2024-09-18T07:35:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:35:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:35:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:16.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:16.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-18T07:35:16.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-18T07:35:16.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-18T07:35:16.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:16 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/3958285469' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:16.932 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:35:16.932 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "10/23 daemons upgraded", 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:35:16.933 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:35:17.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 221760 10.15 MB/sec execute 446 sec latency 1019.720 ms 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN Degraded data redundancy: 201/1578 objects degraded (12.738%), 25 pgs degraded 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 201/1578 objects degraded (12.738%), 25 pgs degraded 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-18T07:35:17.620 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-09-18T07:35:17.621 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-09-18T07:35:17.622 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-09-18T07:35:17.622 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-18T07:35:17.622 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-09-18T07:35:17.622 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1e is active+recovery_wait+undersized+degraded+remapped, acting [2,6] 2024-09-18T07:35:17.622 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-18T07:35:17.622 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.14 is active+undersized+degraded, acting [2,7] 2024-09-18T07:35:17.622 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-18T07:35:17.632 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:17 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 201/1578 objects degraded (12.738%), 25 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:17.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:17 smithi083 ceph-mon[63709]: osdmap e127: 8 total, 8 up, 8 in 2024-09-18T07:35:17.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:17 smithi083 ceph-mon[63709]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:17.633 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:17 smithi083 ceph-mon[63709]: osdmap e128: 8 total, 8 up, 8 in 2024-09-18T07:35:17.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:17 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 201/1578 objects degraded (12.738%), 25 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:17 smithi096 ceph-mon[49483]: osdmap e127: 8 total, 8 up, 8 in 2024-09-18T07:35:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:17 smithi096 ceph-mon[49483]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:17 smithi096 ceph-mon[49483]: osdmap e128: 8 total, 8 up, 8 in 2024-09-18T07:35:18.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 222014 10.14 MB/sec execute 447 sec latency 910.209 ms 2024-09-18T07:35:18.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:18 smithi083 ceph-mon[63709]: pgmap v194: 97 pgs: 1 active+undersized+remapped, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 547 op/s; 185/1647 objects degraded (11.233%); 1012 KiB/s, 37 objects/s recovering 2024-09-18T07:35:18.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:18 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/3506684803' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:35:18.661 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:18 smithi083 ceph-mon[63709]: osdmap e129: 8 total, 8 up, 8 in 2024-09-18T07:35:18.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:18 smithi096 ceph-mon[49483]: pgmap v194: 97 pgs: 1 active+undersized+remapped, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 547 op/s; 185/1647 objects degraded (11.233%); 1012 KiB/s, 37 objects/s recovering 2024-09-18T07:35:18.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:18 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/3506684803' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:35:18.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:18 smithi096 ceph-mon[49483]: osdmap e129: 8 total, 8 up, 8 in 2024-09-18T07:35:19.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 222247 10.12 MB/sec execute 448 sec latency 360.409 ms 2024-09-18T07:35:20.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 222567 10.11 MB/sec execute 449 sec latency 46.254 ms 2024-09-18T07:35:20.730 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:20 smithi083 ceph-mon[63709]: pgmap v196: 97 pgs: 1 active+undersized+remapped, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 6.5 MiB/s wr, 541 op/s; 185/1647 objects degraded (11.233%); 970 KiB/s, 34 objects/s recovering 2024-09-18T07:35:20.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:20 smithi096 ceph-mon[49483]: pgmap v196: 97 pgs: 1 active+undersized+remapped, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 6.5 MiB/s wr, 541 op/s; 185/1647 objects degraded (11.233%); 970 KiB/s, 34 objects/s recovering 2024-09-18T07:35:21.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 222784 10.11 MB/sec execute 450 sec latency 83.009 ms 2024-09-18T07:35:21.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:21 smithi083 ceph-mon[63709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 185/1647 objects degraded (11.233%), 6 pgs degraded) 2024-09-18T07:35:21.743 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:21 smithi083 ceph-mon[63709]: Cluster is now healthy 2024-09-18T07:35:22.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:21 smithi096 ceph-mon[49483]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 185/1647 objects degraded (11.233%), 6 pgs degraded) 2024-09-18T07:35:22.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:21 smithi096 ceph-mon[49483]: Cluster is now healthy 2024-09-18T07:35:22.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 223069 10.10 MB/sec execute 451 sec latency 76.683 ms 2024-09-18T07:35:22.778 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:22 smithi083 ceph-mon[63709]: pgmap v197: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.2 MiB/s wr, 739 op/s; 1.1 MiB/s, 63 objects/s recovering 2024-09-18T07:35:23.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:22 smithi096 ceph-mon[49483]: pgmap v197: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.2 MiB/s wr, 739 op/s; 1.1 MiB/s, 63 objects/s recovering 2024-09-18T07:35:23.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 223451 10.11 MB/sec execute 452 sec latency 41.755 ms 2024-09-18T07:35:24.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 223936 10.13 MB/sec execute 453 sec latency 38.637 ms 2024-09-18T07:35:24.809 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:24 smithi083 ceph-mon[63709]: pgmap v198: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 2.5 MiB/s wr, 323 op/s; 374 KiB/s, 34 objects/s recovering 2024-09-18T07:35:25.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:24 smithi096 ceph-mon[49483]: pgmap v198: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 2.5 MiB/s wr, 323 op/s; 374 KiB/s, 34 objects/s recovering 2024-09-18T07:35:25.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 224297 10.13 MB/sec execute 454 sec latency 63.757 ms 2024-09-18T07:35:26.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 224626 10.13 MB/sec execute 455 sec latency 49.554 ms 2024-09-18T07:35:26.879 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:26 smithi083 ceph-mon[63709]: pgmap v199: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.2 MiB/s wr, 285 op/s; 326 KiB/s, 30 objects/s recovering 2024-09-18T07:35:27.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:26 smithi096 ceph-mon[49483]: pgmap v199: 97 pgs: 97 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.2 MiB/s wr, 285 op/s; 326 KiB/s, 30 objects/s recovering 2024-09-18T07:35:27.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 224879 10.12 MB/sec execute 456 sec latency 56.230 ms 2024-09-18T07:35:28.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 225107 10.10 MB/sec execute 457 sec latency 55.712 ms 2024-09-18T07:35:28.669 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:28 smithi083 ceph-mon[63709]: pgmap v200: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 453 op/s; 262 KiB/s, 24 objects/s recovering 2024-09-18T07:35:29.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:28 smithi096 ceph-mon[49483]: pgmap v200: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 453 op/s; 262 KiB/s, 24 objects/s recovering 2024-09-18T07:35:29.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 225499 10.10 MB/sec execute 458 sec latency 41.761 ms 2024-09-18T07:35:30.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 226014 10.10 MB/sec execute 459 sec latency 38.397 ms 2024-09-18T07:35:30.683 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:30 smithi083 ceph-mon[63709]: pgmap v201: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 411 op/s; 238 KiB/s, 21 objects/s recovering 2024-09-18T07:35:31.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:30 smithi096 ceph-mon[49483]: pgmap v201: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 411 op/s; 238 KiB/s, 21 objects/s recovering 2024-09-18T07:35:31.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 226405 10.10 MB/sec execute 460 sec latency 36.763 ms 2024-09-18T07:35:31.973 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:31.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:35:31.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-18T07:35:31.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:31 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-18T07:35:31.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:31 smithi083 ceph-mon[63709]: Upgrade: osd.4 is safe to restart 2024-09-18T07:35:31.974 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:35:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-18T07:35:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:31 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-18T07:35:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:31 smithi096 ceph-mon[49483]: Upgrade: osd.4 is safe to restart 2024-09-18T07:35:32.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:32.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 226949 10.11 MB/sec execute 461 sec latency 27.642 ms 2024-09-18T07:35:33.241 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:33 smithi083 ceph-mon[63709]: pgmap v202: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 608 op/s; 218 KiB/s, 20 objects/s recovering 2024-09-18T07:35:33.242 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:33 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:33.242 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:33 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-18T07:35:33.242 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:33 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:35:33.243 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:33 smithi096 ceph-mon[49483]: pgmap v202: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 608 op/s; 218 KiB/s, 20 objects/s recovering 2024-09-18T07:35:33.243 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:33 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:33.243 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:33 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-18T07:35:33.243 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:33 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:35:33.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 227365 10.12 MB/sec execute 462 sec latency 31.306 ms 2024-09-18T07:35:34.248 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:34 smithi083 ceph-mon[63709]: Upgrade: Updating osd.4 2024-09-18T07:35:34.248 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:34 smithi083 ceph-mon[63709]: Deploying daemon osd.4 on smithi096 2024-09-18T07:35:34.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:34 smithi096 ceph-mon[49483]: Upgrade: Updating osd.4 2024-09-18T07:35:34.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:34 smithi096 ceph-mon[49483]: Deploying daemon osd.4 on smithi096 2024-09-18T07:35:34.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 227722 10.14 MB/sec execute 463 sec latency 53.409 ms 2024-09-18T07:35:35.269 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:35 smithi096 ceph-mon[49483]: pgmap v203: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-18T07:35:35.288 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:35 smithi083 ceph-mon[63709]: pgmap v203: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-18T07:35:35.601 INFO:teuthology.orchestra.run.smithi083.stdout: 5 228072 10.14 MB/sec execute 464 sec latency 38.291 ms 2024-09-18T07:35:36.372 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:36 smithi096 systemd[1]: Stopping Ceph osd.4 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:35:36.603 INFO:teuthology.orchestra.run.smithi083.stdout: 5 228315 10.12 MB/sec execute 465 sec latency 68.254 ms 2024-09-18T07:35:36.871 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:36 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[30068]: 2024-09-18T07:35:36.417+0000 7f43e8ecc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:35:36.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:36 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[30068]: 2024-09-18T07:35:36.417+0000 7f43e8ecc700 -1 osd.4 129 *** Got signal Terminated *** 2024-09-18T07:35:36.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:36 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[30068]: 2024-09-18T07:35:36.417+0000 7f43e8ecc700 -1 osd.4 129 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:35:37.091 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:37 smithi083 ceph-mon[63709]: pgmap v204: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 420 op/s 2024-09-18T07:35:37.091 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:37 smithi083 ceph-mon[63709]: osd.4 marked itself down and dead 2024-09-18T07:35:37.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:37 smithi096 ceph-mon[49483]: pgmap v204: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 420 op/s 2024-09-18T07:35:37.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:37 smithi096 ceph-mon[49483]: osd.4 marked itself down and dead 2024-09-18T07:35:37.603 INFO:teuthology.orchestra.run.smithi083.stdout: 5 228497 10.10 MB/sec execute 466 sec latency 485.031 ms 2024-09-18T07:35:37.959 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:37 smithi096 podman[52897]: 2024-09-18 07:35:37.687736765 +0000 UTC m=+1.402414704 container died 305fa8f943da6b54f5ed0218d5c57f4f124d95c157d405c0a92ecaeeae3fd5a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4, version=8, architecture=x86_64, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, release=754, vcs-type=git, name=centos-stream, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-18T07:35:38.314 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:38 smithi083 ceph-mon[63709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:35:38.314 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:38 smithi083 ceph-mon[63709]: osdmap e130: 8 total, 7 up, 8 in 2024-09-18T07:35:38.336 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:38 smithi096 ceph-mon[49483]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:35:38.336 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:38 smithi096 ceph-mon[49483]: osdmap e130: 8 total, 7 up, 8 in 2024-09-18T07:35:38.336 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:38 smithi096 podman[52897]: 2024-09-18 07:35:38.048724094 +0000 UTC m=+1.763402028 container cleanup 305fa8f943da6b54f5ed0218d5c57f4f124d95c157d405c0a92ecaeeae3fd5a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, vcs-type=git, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD) 2024-09-18T07:35:38.336 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:38 smithi096 bash[52897]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4 2024-09-18T07:35:38.603 INFO:teuthology.orchestra.run.smithi083.stdout: 5 228596 10.09 MB/sec execute 467 sec latency 986.728 ms 2024-09-18T07:35:38.621 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:38 smithi096 podman[52939]: 2024-09-18 07:35:38.334354901 +0000 UTC m=+0.643861981 container remove 305fa8f943da6b54f5ed0218d5c57f4f124d95c157d405c0a92ecaeeae3fd5a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, release=754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.buildah.version=1.19.8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux ) 2024-09-18T07:35:38.982 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:38 smithi096 podman[53020]: 2024-09-18 07:35:38.598457386 +0000 UTC m=+0.019421533 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:35:38.983 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:38 smithi096 podman[53020]: 2024-09-18 07:35:38.71885529 +0000 UTC m=+0.139819418 container create d32b002c7900d0d866f8b84a32d027164930b503aa17e6247c26a3fc31781c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-18T07:35:38.983 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:38 smithi096 podman[53020]: 2024-09-18 07:35:38.980466753 +0000 UTC m=+0.401430892 container init d32b002c7900d0d866f8b84a32d027164930b503aa17e6247c26a3fc31781c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:35:39.072 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:39 smithi083 ceph-mon[63709]: pgmap v206: 97 pgs: 14 stale+active+clean, 83 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 570 op/s 2024-09-18T07:35:39.073 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:39 smithi083 ceph-mon[63709]: osdmap e131: 8 total, 7 up, 8 in 2024-09-18T07:35:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:39 smithi096 ceph-mon[49483]: pgmap v206: 97 pgs: 14 stale+active+clean, 83 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 570 op/s 2024-09-18T07:35:39.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:39 smithi096 ceph-mon[49483]: osdmap e131: 8 total, 7 up, 8 in 2024-09-18T07:35:39.372 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:38 smithi096 podman[53020]: 2024-09-18 07:35:38.986422716 +0000 UTC m=+0.407386841 container start d32b002c7900d0d866f8b84a32d027164930b503aa17e6247c26a3fc31781c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-18T07:35:39.372 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:39 smithi096 podman[53020]: 2024-09-18 07:35:39.088882219 +0000 UTC m=+0.509846350 container attach d32b002c7900d0d866f8b84a32d027164930b503aa17e6247c26a3fc31781c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:35:39.373 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:39 smithi096 podman[53020]: 2024-09-18 07:35:39.199982886 +0000 UTC m=+0.620947013 container died d32b002c7900d0d866f8b84a32d027164930b503aa17e6247c26a3fc31781c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-09-18T07:35:39.603 INFO:teuthology.orchestra.run.smithi083.stdout: 5 228991 10.09 MB/sec execute 468 sec latency 40.027 ms 2024-09-18T07:35:40.143 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:39 smithi096 podman[53043]: 2024-09-18 07:35:39.877013762 +0000 UTC m=+0.670270094 container remove d32b002c7900d0d866f8b84a32d027164930b503aa17e6247c26a3fc31781c3e (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-18T07:35:40.143 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:39 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-18T07:35:40.144 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:39 smithi096 systemd[1]: Stopped Ceph osd.4 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:35:40.144 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:39 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.4.service: Consumed 4min 5.773s CPU time. 2024-09-18T07:35:40.414 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 systemd[1]: Starting Ceph osd.4 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:35:40.414 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 podman[53140]: 2024-09-18 07:35:40.282570972 +0000 UTC m=+0.025963339 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:35:40.603 INFO:teuthology.orchestra.run.smithi083.stdout: 5 229505 10.09 MB/sec execute 469 sec latency 43.473 ms 2024-09-18T07:35:40.667 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 podman[53140]: 2024-09-18 07:35:40.412220035 +0000 UTC m=+0.155612380 container create 69bc300e7d0dcbde38f41044525bc0fc2de3e698f52d8aa36c28246925bb7ee7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-18T07:35:40.667 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 podman[53140]: 2024-09-18 07:35:40.665969437 +0000 UTC m=+0.409361807 container init 69bc300e7d0dcbde38f41044525bc0fc2de3e698f52d8aa36c28246925bb7ee7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5) 2024-09-18T07:35:41.069 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 podman[53140]: 2024-09-18 07:35:40.670475709 +0000 UTC m=+0.413868053 container start 69bc300e7d0dcbde38f41044525bc0fc2de3e698f52d8aa36c28246925bb7ee7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-18T07:35:41.069 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 podman[53140]: 2024-09-18 07:35:40.77501317 +0000 UTC m=+0.518405517 container attach 69bc300e7d0dcbde38f41044525bc0fc2de3e698f52d8aa36c28246925bb7ee7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-18T07:35:41.069 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate[53155]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:35:41.069 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 bash[53140]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:35:41.069 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate[53155]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:35:41.069 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:40 smithi096 bash[53140]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:35:41.094 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:41 smithi083 ceph-mon[63709]: pgmap v208: 97 pgs: 14 stale+active+clean, 83 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.2 MiB/s wr, 366 op/s 2024-09-18T07:35:41.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:41 smithi096 ceph-mon[49483]: pgmap v208: 97 pgs: 14 stale+active+clean, 83 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.2 MiB/s wr, 366 op/s 2024-09-18T07:35:41.603 INFO:teuthology.orchestra.run.smithi083.stdout: 5 229983 10.08 MB/sec execute 470 sec latency 48.659 ms 2024-09-18T07:35:42.353 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:42 smithi083 ceph-mon[63709]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 681 op/s; 243/1596 objects degraded (15.226%) 2024-09-18T07:35:42.353 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:42 smithi083 ceph-mon[63709]: Health check failed: Degraded data redundancy: 243/1596 objects degraded (15.226%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:42.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:42 smithi096 ceph-mon[49483]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 681 op/s; 243/1596 objects degraded (15.226%) 2024-09-18T07:35:42.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:42 smithi096 ceph-mon[49483]: Health check failed: Degraded data redundancy: 243/1596 objects degraded (15.226%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:42.603 INFO:teuthology.orchestra.run.smithi083.stdout: 5 230568 10.11 MB/sec execute 471 sec latency 43.278 ms 2024-09-18T07:35:43.472 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate[53155]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-18T07:35:43.472 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 bash[53140]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-18T07:35:43.472 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate[53155]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-18T07:35:43.472 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 bash[53140]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-18T07:35:43.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 231164 10.14 MB/sec execute 472 sec latency 31.433 ms 2024-09-18T07:35:43.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate[53155]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-18T07:35:43.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 bash[53140]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-18T07:35:43.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate[53155]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-18T07:35:43.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 bash[53140]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-18T07:35:43.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate[53155]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-18T07:35:43.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 bash[53140]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-18T07:35:43.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate[53155]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-18T07:35:43.872 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 bash[53140]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-18T07:35:43.873 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate[53155]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-18T07:35:43.873 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 bash[53140]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-18T07:35:43.873 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:43 smithi096 podman[53140]: 2024-09-18 07:35:43.503050664 +0000 UTC m=+3.246443023 container died 69bc300e7d0dcbde38f41044525bc0fc2de3e698f52d8aa36c28246925bb7ee7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-18T07:35:44.450 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:44 smithi096 podman[53321]: 2024-09-18 07:35:44.196951691 +0000 UTC m=+0.687042616 container remove 69bc300e7d0dcbde38f41044525bc0fc2de3e698f52d8aa36c28246925bb7ee7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:35:44.450 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:44 smithi096 podman[53379]: 2024-09-18 07:35:44.325639675 +0000 UTC m=+0.016432862 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:35:44.450 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:44 smithi096 podman[53379]: 2024-09-18 07:35:44.448115637 +0000 UTC m=+0.138908822 container create 2ca3dcb5baa0011957d859d007072eff7fb29205ecec4d5bcdac20a7afc3e7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-18T07:35:44.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 231535 10.14 MB/sec execute 473 sec latency 34.399 ms 2024-09-18T07:35:44.722 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:44 smithi096 ceph-mon[49483]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.5 MiB/s wr, 679 op/s; 243/1596 objects degraded (15.226%) 2024-09-18T07:35:44.722 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:44 smithi096 podman[53379]: 2024-09-18 07:35:44.693000598 +0000 UTC m=+0.383793801 container init 2ca3dcb5baa0011957d859d007072eff7fb29205ecec4d5bcdac20a7afc3e7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True) 2024-09-18T07:35:44.722 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:44 smithi096 podman[53379]: 2024-09-18 07:35:44.69678108 +0000 UTC m=+0.387574272 container start 2ca3dcb5baa0011957d859d007072eff7fb29205ecec4d5bcdac20a7afc3e7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-09-18T07:35:44.877 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:44 smithi083 ceph-mon[63709]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.5 MiB/s wr, 679 op/s; 243/1596 objects degraded (15.226%) 2024-09-18T07:35:45.122 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:44 smithi096 ceph-osd[53398]: -- 172.21.15.96:0/3125662516 <== mon.1 v2:172.21.15.96:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ab24e141e0 con 0x55ab23fafc00 2024-09-18T07:35:45.122 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:44 smithi096 bash[53379]: 2ca3dcb5baa0011957d859d007072eff7fb29205ecec4d5bcdac20a7afc3e7b7 2024-09-18T07:35:45.122 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:44 smithi096 systemd[1]: Started Ceph osd.4 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:35:45.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 231903 10.13 MB/sec execute 474 sec latency 34.411 ms 2024-09-18T07:35:45.888 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:45 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:45.889 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:45 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:45.889 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:45 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:35:46.026 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:45 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:46.027 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:45 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:46.027 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:45 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:35:46.603 INFO:teuthology.orchestra.run.smithi083.stdout: 5 232219 10.12 MB/sec execute 475 sec latency 54.020 ms 2024-09-18T07:35:46.871 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:46 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:35:46.585+0000 7f52b37f9640 -1 Falling back to public interface 2024-09-18T07:35:47.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:46 smithi083 ceph-mon[63709]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 2.5 MiB/s wr, 350 op/s; 246/1605 objects degraded (15.327%) 2024-09-18T07:35:47.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:47.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:35:47.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:46 smithi096 ceph-mon[49483]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 2.5 MiB/s wr, 350 op/s; 246/1605 objects degraded (15.327%) 2024-09-18T07:35:47.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:47.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:35:47.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 232614 10.13 MB/sec execute 476 sec latency 76.105 ms 2024-09-18T07:35:48.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:47 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 246/1605 objects degraded (15.327%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:48.195 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:35:48.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:47 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 246/1605 objects degraded (15.327%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:48.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 233021 10.12 MB/sec execute 477 sec latency 44.490 ms 2024-09-18T07:35:49.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:48 smithi096 ceph-mon[49483]: pgmap v212: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 520 op/s; 274/1710 objects degraded (16.023%) 2024-09-18T07:35:49.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:48 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:49.202 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:48 smithi083 ceph-mon[63709]: pgmap v212: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 520 op/s; 274/1710 objects degraded (16.023%) 2024-09-18T07:35:49.203 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:48 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:49.303 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:35:49.303 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (7m) 40s ago 16m 20.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:35:49.303 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (3m) 40s ago 16m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:35:49.303 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (3m) 0s ago 15m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:35:49.303 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (7m) 40s ago 16m 45.0M - dad864ee21e9 06ff04a18f50 2024-09-18T07:35:49.303 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (10m) 40s ago 10m 41.2M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:35:49.303 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (10m) 0s ago 10m 16.7M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (6m) 40s ago 17m 520M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (5m) 0s ago 15m 457M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (5m) 40s ago 17m 51.7M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (4m) 0s ago 15m 41.5M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (10m) 40s ago 10m 279M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (10m) 0s ago 10m 90.6M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (6m) 40s ago 16m 9.79M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (6m) 0s ago 15m 11.9M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3m) 40s ago 14m 365M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:35:49.304 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (2m) 40s ago 14m 277M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:35:49.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (101s) 40s ago 13m 212M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:35:49.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (51s) 40s ago 13m 11.7M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:35:49.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (4s) 0s ago 13m 14.3M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2ca3dcb5baa0 2024-09-18T07:35:49.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (12m) 0s ago 12m 548M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:35:49.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (12m) 0s ago 12m 474M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:35:49.305 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (11m) 0s ago 11m 759M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:35:49.305 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (5m) 40s ago 15m 53.3M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:35:49.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 233392 10.11 MB/sec execute 478 sec latency 49.732 ms 2024-09-18T07:35:50.035 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 4 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:35:50.036 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:35:50.037 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:35:50.037 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:35:50.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-18T07:35:50.037 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 8 2024-09-18T07:35:50.037 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:35:50.037 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:35:50.218 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:49 smithi083 ceph-mon[63709]: from='client.44277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:50.219 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:49 smithi083 ceph-mon[63709]: from='client.44281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:50.219 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:49 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:50.219 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:49 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:50.219 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:49 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:50.228 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:49 smithi096 ceph-mon[49483]: from='client.44277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:50.228 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:49 smithi096 ceph-mon[49483]: from='client.44281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:50.228 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:49 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:50.228 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:49 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:50.228 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:49 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:50.551 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "11/23 daemons upgraded", 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:35:50.552 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:35:50.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 233750 10.11 MB/sec execute 479 sec latency 42.794 ms 2024-09-18T07:35:50.982 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:50 smithi083 ceph-mon[63709]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:50.983 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:50 smithi083 ceph-mon[63709]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 451 op/s; 274/1710 objects degraded (16.023%) 2024-09-18T07:35:50.983 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:50 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/2008204445' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:51.166 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 274/1710 objects degraded (16.023%), 28 pgs degraded 2024-09-18T07:35:51.166 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-18T07:35:51.166 INFO:teuthology.orchestra.run.smithi083.stdout: osd.4 (root=default,host=smithi096) is down 2024-09-18T07:35:51.166 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 274/1710 objects degraded (16.023%), 28 pgs degraded 2024-09-18T07:35:51.166 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-18T07:35:51.166 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-18T07:35:51.166 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1f is active+undersized+degraded, acting [0,2] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-18T07:35:51.167 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-18T07:35:51.168 INFO:teuthology.orchestra.run.smithi083.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-18T07:35:51.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:50 smithi096 ceph-mon[49483]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:51.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:50 smithi096 ceph-mon[49483]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 451 op/s; 274/1710 objects degraded (16.023%) 2024-09-18T07:35:51.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:50 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/2008204445' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:51.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 234152 10.12 MB/sec execute 480 sec latency 31.076 ms 2024-09-18T07:35:52.006 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:51 smithi083 ceph-mon[63709]: from='client.44293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:52.007 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:51 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/3247140585' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:35:52.301 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:51 smithi096 ceph-mon[49483]: from='client.44293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:35:52.301 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:51 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/3247140585' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:35:52.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 234554 10.13 MB/sec execute 481 sec latency 39.386 ms 2024-09-18T07:35:53.015 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:52 smithi083 ceph-mon[63709]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 701 op/s; 238/1575 objects degraded (15.111%) 2024-09-18T07:35:53.015 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:52 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 238/1575 objects degraded (15.111%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:53.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:52 smithi096 ceph-mon[49483]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 701 op/s; 238/1575 objects degraded (15.111%) 2024-09-18T07:35:53.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:52 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 238/1575 objects degraded (15.111%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:53.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 234958 10.14 MB/sec execute 482 sec latency 43.062 ms 2024-09-18T07:35:54.371 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:35:54.107+0000 7f52b37f9640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-18T07:35:54.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235236 10.14 MB/sec execute 483 sec latency 46.506 ms 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: pgmap v215: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 491 op/s; 238/1575 objects degraded (15.111%) 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:55.054 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:55.055 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-18T07:35:55.371 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:55 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:35:55.156+0000 7f52b37f9640 -1 osd.4 129 log_to_monitors true 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: pgmap v215: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 491 op/s; 238/1575 objects degraded (15.111%) 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:35:55.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-18T07:35:55.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235495 10.12 MB/sec execute 484 sec latency 43.609 ms 2024-09-18T07:35:56.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:55 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-18T07:35:56.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:55 smithi083 ceph-mon[63709]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-18T07:35:56.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:55 smithi083 ceph-mon[63709]: from='osd.4 [v2:172.21.15.96:6800/2498510594,v1:172.21.15.96:6801/2498510594]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-18T07:35:56.105 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:55 smithi083 ceph-mon[63709]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-18T07:35:56.371 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:35:56 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:35:56.007+0000 7f52ab5a3640 -1 osd.4 129 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:35:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:55 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-18T07:35:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:55 smithi096 ceph-mon[49483]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-18T07:35:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:55 smithi096 ceph-mon[49483]: from='osd.4 [v2:172.21.15.96:6800/2498510594,v1:172.21.15.96:6801/2498510594]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-18T07:35:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:55 smithi096 ceph-mon[49483]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-18T07:35:56.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235692 10.11 MB/sec execute 485 sec latency 49.012 ms 2024-09-18T07:35:57.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:56 smithi083 ceph-mon[63709]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.2 MiB/s wr, 495 op/s; 241/1584 objects degraded (15.215%) 2024-09-18T07:35:57.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:56 smithi083 ceph-mon[63709]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-18T07:35:57.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:56 smithi083 ceph-mon[63709]: from='osd.4 [v2:172.21.15.96:6800/2498510594,v1:172.21.15.96:6801/2498510594]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:35:57.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:56 smithi083 ceph-mon[63709]: osdmap e132: 8 total, 7 up, 8 in 2024-09-18T07:35:57.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:56 smithi083 ceph-mon[63709]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:35:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:56 smithi096 ceph-mon[49483]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.2 MiB/s wr, 495 op/s; 241/1584 objects degraded (15.215%) 2024-09-18T07:35:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:56 smithi096 ceph-mon[49483]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-18T07:35:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:56 smithi096 ceph-mon[49483]: from='osd.4 [v2:172.21.15.96:6800/2498510594,v1:172.21.15.96:6801/2498510594]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:35:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:56 smithi096 ceph-mon[49483]: osdmap e132: 8 total, 7 up, 8 in 2024-09-18T07:35:57.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:56 smithi096 ceph-mon[49483]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:35:57.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235781 10.09 MB/sec execute 486 sec latency 618.351 ms 2024-09-18T07:35:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:57 smithi083 ceph-mon[63709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:35:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:57 smithi083 ceph-mon[63709]: osd.4 [v2:172.21.15.96:6800/2498510594,v1:172.21.15.96:6801/2498510594] boot 2024-09-18T07:35:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:57 smithi083 ceph-mon[63709]: osdmap e133: 8 total, 8 up, 8 in 2024-09-18T07:35:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:57 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-18T07:35:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:57 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 241/1584 objects degraded (15.215%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:57 smithi083 ceph-mon[63709]: osdmap e134: 8 total, 8 up, 8 in 2024-09-18T07:35:58.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:57 smithi096 ceph-mon[49483]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:35:58.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:57 smithi096 ceph-mon[49483]: osd.4 [v2:172.21.15.96:6800/2498510594,v1:172.21.15.96:6801/2498510594] boot 2024-09-18T07:35:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:57 smithi096 ceph-mon[49483]: osdmap e133: 8 total, 8 up, 8 in 2024-09-18T07:35:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:57 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-18T07:35:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:57 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 241/1584 objects degraded (15.215%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:35:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:57 smithi096 ceph-mon[49483]: osdmap e134: 8 total, 8 up, 8 in 2024-09-18T07:35:58.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235781 10.07 MB/sec execute 487 sec latency 1618.384 ms 2024-09-18T07:35:59.178 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:58 smithi083 ceph-mon[63709]: pgmap v220: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.9 MiB/s wr, 434 op/s; 251/1620 objects degraded (15.494%) 2024-09-18T07:35:59.179 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:35:58 smithi083 ceph-mon[63709]: osdmap e135: 8 total, 8 up, 8 in 2024-09-18T07:35:59.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:58 smithi096 ceph-mon[49483]: pgmap v220: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.9 MiB/s wr, 434 op/s; 251/1620 objects degraded (15.494%) 2024-09-18T07:35:59.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:35:58 smithi096 ceph-mon[49483]: osdmap e135: 8 total, 8 up, 8 in 2024-09-18T07:35:59.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235960 10.06 MB/sec execute 488 sec latency 1705.552 ms 2024-09-18T07:36:00.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 235960 10.04 MB/sec execute 489 sec latency 1056.355 ms 2024-09-18T07:36:00.842 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:00 smithi083 ceph-mon[63709]: osdmap e136: 8 total, 8 up, 8 in 2024-09-18T07:36:00.843 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:00 smithi083 ceph-mon[63709]: pgmap v223: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s wr, 170 op/s; 251/1620 objects degraded (15.494%) 2024-09-18T07:36:00.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:00 smithi096 ceph-mon[49483]: osdmap e136: 8 total, 8 up, 8 in 2024-09-18T07:36:00.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:00 smithi096 ceph-mon[49483]: pgmap v223: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s wr, 170 op/s; 251/1620 objects degraded (15.494%) 2024-09-18T07:36:01.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 236161 10.03 MB/sec execute 490 sec latency 1066.746 ms 2024-09-18T07:36:01.629 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:01 smithi083 ceph-mon[63709]: osdmap e137: 8 total, 8 up, 8 in 2024-09-18T07:36:01.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:01.630 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:36:01.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:01 smithi096 ceph-mon[49483]: osdmap e137: 8 total, 8 up, 8 in 2024-09-18T07:36:01.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:01.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:36:02.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 236421 10.02 MB/sec execute 491 sec latency 570.318 ms 2024-09-18T07:36:02.674 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:02 smithi083 ceph-mon[63709]: pgmap v225: 97 pgs: 2 peering, 9 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 83 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 1.7 MiB/s wr, 222 op/s; 245/1374 objects degraded (17.831%); 2.5 MiB/s, 86 keys/s, 42 objects/s recovering 2024-09-18T07:36:02.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:02 smithi083 ceph-mon[63709]: osdmap e138: 8 total, 8 up, 8 in 2024-09-18T07:36:02.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:02 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 251/1620 objects degraded (15.494%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:02.675 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:02 smithi083 ceph-mon[63709]: osdmap e139: 8 total, 8 up, 8 in 2024-09-18T07:36:03.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:02 smithi096 ceph-mon[49483]: pgmap v225: 97 pgs: 2 peering, 9 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 83 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 1.7 MiB/s wr, 222 op/s; 245/1374 objects degraded (17.831%); 2.5 MiB/s, 86 keys/s, 42 objects/s recovering 2024-09-18T07:36:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:02 smithi096 ceph-mon[49483]: osdmap e138: 8 total, 8 up, 8 in 2024-09-18T07:36:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:02 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 251/1620 objects degraded (15.494%), 28 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:02 smithi096 ceph-mon[49483]: osdmap e139: 8 total, 8 up, 8 in 2024-09-18T07:36:03.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 236857 10.02 MB/sec execute 492 sec latency 661.887 ms 2024-09-18T07:36:04.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 237152 10.01 MB/sec execute 493 sec latency 897.197 ms 2024-09-18T07:36:04.756 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:04 smithi083 ceph-mon[63709]: osdmap e140: 8 total, 8 up, 8 in 2024-09-18T07:36:04.757 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:04 smithi083 ceph-mon[63709]: pgmap v229: 97 pgs: 2 peering, 9 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 83 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 1.7 MiB/s wr, 223 op/s; 245/1374 objects degraded (17.831%); 2.5 MiB/s, 86 keys/s, 42 objects/s recovering 2024-09-18T07:36:04.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:04 smithi096 ceph-mon[49483]: osdmap e140: 8 total, 8 up, 8 in 2024-09-18T07:36:04.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:04 smithi096 ceph-mon[49483]: pgmap v229: 97 pgs: 2 peering, 9 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 83 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 1.7 MiB/s wr, 223 op/s; 245/1374 objects degraded (17.831%); 2.5 MiB/s, 86 keys/s, 42 objects/s recovering 2024-09-18T07:36:05.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 237494 10.01 MB/sec execute 494 sec latency 800.536 ms 2024-09-18T07:36:05.787 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:05 smithi083 ceph-mon[63709]: osdmap e141: 8 total, 8 up, 8 in 2024-09-18T07:36:05.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:05 smithi096 ceph-mon[49483]: osdmap e141: 8 total, 8 up, 8 in 2024-09-18T07:36:06.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 237681 10.00 MB/sec execute 495 sec latency 856.722 ms 2024-09-18T07:36:06.801 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:06 smithi083 ceph-mon[63709]: osdmap e142: 8 total, 8 up, 8 in 2024-09-18T07:36:06.802 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:06 smithi083 ceph-mon[63709]: pgmap v232: 97 pgs: 2 peering, 9 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 83 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.3 KiB/s wr, 5 op/s; 245/1374 objects degraded (17.831%) 2024-09-18T07:36:06.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:06 smithi096 ceph-mon[49483]: osdmap e142: 8 total, 8 up, 8 in 2024-09-18T07:36:06.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:06 smithi096 ceph-mon[49483]: pgmap v232: 97 pgs: 2 peering, 9 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 83 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.3 KiB/s wr, 5 op/s; 245/1374 objects degraded (17.831%) 2024-09-18T07:36:07.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 238074 10.02 MB/sec execute 496 sec latency 852.840 ms 2024-09-18T07:36:07.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:07 smithi083 ceph-mon[63709]: osdmap e143: 8 total, 8 up, 8 in 2024-09-18T07:36:07.815 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:07 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 245/1374 objects degraded (17.831%), 12 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:07.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:07 smithi096 ceph-mon[49483]: osdmap e143: 8 total, 8 up, 8 in 2024-09-18T07:36:07.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:07 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 245/1374 objects degraded (17.831%), 12 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:08.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 238402 10.02 MB/sec execute 497 sec latency 52.510 ms 2024-09-18T07:36:08.841 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:08 smithi083 ceph-mon[63709]: pgmap v234: 97 pgs: 5 peering, 92 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.5 MiB/s wr, 494 op/s; 317 KiB/s, 62 objects/s recovering 2024-09-18T07:36:08.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:08 smithi096 ceph-mon[49483]: pgmap v234: 97 pgs: 5 peering, 92 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.5 MiB/s wr, 494 op/s; 317 KiB/s, 62 objects/s recovering 2024-09-18T07:36:09.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 238679 10.02 MB/sec execute 498 sec latency 61.530 ms 2024-09-18T07:36:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:09 smithi083 ceph-mon[63709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 245/1374 objects degraded (17.831%), 12 pgs degraded) 2024-09-18T07:36:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:09 smithi083 ceph-mon[63709]: Cluster is now healthy 2024-09-18T07:36:09.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:09 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-18T07:36:09.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:09 smithi096 ceph-mon[49483]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 245/1374 objects degraded (17.831%), 12 pgs degraded) 2024-09-18T07:36:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:09 smithi096 ceph-mon[49483]: Cluster is now healthy 2024-09-18T07:36:09.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:09 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-18T07:36:10.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 238908 10.01 MB/sec execute 499 sec latency 82.117 ms 2024-09-18T07:36:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:10 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-18T07:36:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:10 smithi083 ceph-mon[63709]: Upgrade: osd.5 is safe to restart 2024-09-18T07:36:10.643 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:10 smithi083 ceph-mon[63709]: pgmap v235: 97 pgs: 5 peering, 92 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.0 MiB/s wr, 331 op/s; 212 KiB/s, 41 objects/s recovering 2024-09-18T07:36:10.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:10 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-18T07:36:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:10 smithi096 ceph-mon[49483]: Upgrade: osd.5 is safe to restart 2024-09-18T07:36:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:10 smithi096 ceph-mon[49483]: pgmap v235: 97 pgs: 5 peering, 92 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.0 MiB/s wr, 331 op/s; 212 KiB/s, 41 objects/s recovering 2024-09-18T07:36:11.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 239185 9.99 MB/sec execute 500 sec latency 43.308 ms 2024-09-18T07:36:12.433 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:12 smithi083 ceph-mon[63709]: Upgrade: Updating osd.5 2024-09-18T07:36:12.434 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:12.434 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-18T07:36:12.434 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:36:12.434 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:12 smithi083 ceph-mon[63709]: Deploying daemon osd.5 on smithi096 2024-09-18T07:36:12.434 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:12 smithi083 ceph-mon[63709]: pgmap v236: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 6.0 MiB/s wr, 546 op/s; 182 KiB/s, 38 objects/s recovering 2024-09-18T07:36:12.590 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:12 smithi096 ceph-mon[49483]: Upgrade: Updating osd.5 2024-09-18T07:36:12.590 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:12.590 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-18T07:36:12.590 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:36:12.590 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:12 smithi096 ceph-mon[49483]: Deploying daemon osd.5 on smithi096 2024-09-18T07:36:12.590 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:12 smithi096 ceph-mon[49483]: pgmap v236: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 6.0 MiB/s wr, 546 op/s; 182 KiB/s, 38 objects/s recovering 2024-09-18T07:36:12.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 239430 9.99 MB/sec execute 501 sec latency 116.330 ms 2024-09-18T07:36:13.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 239782 9.98 MB/sec execute 502 sec latency 41.348 ms 2024-09-18T07:36:14.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 240147 9.97 MB/sec execute 503 sec latency 53.390 ms 2024-09-18T07:36:14.737 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:14 smithi083 ceph-mon[63709]: pgmap v237: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 5.3 MiB/s wr, 476 op/s; 159 KiB/s, 33 objects/s recovering 2024-09-18T07:36:14.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:14 smithi096 ceph-mon[49483]: pgmap v237: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 5.3 MiB/s wr, 476 op/s; 159 KiB/s, 33 objects/s recovering 2024-09-18T07:36:15.371 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:15 smithi096 systemd[1]: Stopping Ceph osd.5 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:36:15.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 240468 9.97 MB/sec execute 504 sec latency 55.860 ms 2024-09-18T07:36:15.775 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:15 smithi083 ceph-mon[63709]: osd.5 marked itself down and dead 2024-09-18T07:36:15.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:15 smithi096 ceph-mon[49483]: osd.5 marked itself down and dead 2024-09-18T07:36:15.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:15 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[33022]: 2024-09-18T07:36:15.519+0000 7ff16ac17700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:36:15.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:15 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[33022]: 2024-09-18T07:36:15.519+0000 7ff16ac17700 -1 osd.5 143 *** Got signal Terminated *** 2024-09-18T07:36:15.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:15 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[33022]: 2024-09-18T07:36:15.519+0000 7ff16ac17700 -1 osd.5 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:36:16.557 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:16 smithi096 podman[54947]: 2024-09-18 07:36:16.293566164 +0000 UTC m=+0.894112143 container died a8dc7a20676a87b8bf3df7d43ef513533bc73a8f86ceecee74ddb5d49ca6fed1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-type=git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_CLEAN=True, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, release=754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-18T07:36:16.604 INFO:teuthology.orchestra.run.smithi083.stdout: 5 240569 9.95 MB/sec execute 505 sec latency 949.542 ms 2024-09-18T07:36:16.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:16 smithi096 ceph-mon[49483]: pgmap v238: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.9 MiB/s wr, 531 op/s; 127 KiB/s, 27 objects/s recovering 2024-09-18T07:36:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:16 smithi096 ceph-mon[49483]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:36:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:16 smithi096 ceph-mon[49483]: osdmap e144: 8 total, 7 up, 8 in 2024-09-18T07:36:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:16.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:36:16.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:16 smithi096 podman[54947]: 2024-09-18 07:36:16.645679352 +0000 UTC m=+1.246225326 container cleanup a8dc7a20676a87b8bf3df7d43ef513533bc73a8f86ceecee74ddb5d49ca6fed1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.expose-services=, name=centos-stream, version=8, GIT_BRANCH=HEAD, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_CLEAN=True, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-18T07:36:16.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:16 smithi096 bash[54947]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5 2024-09-18T07:36:16.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:16 smithi083 ceph-mon[63709]: pgmap v238: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.9 MiB/s wr, 531 op/s; 127 KiB/s, 27 objects/s recovering 2024-09-18T07:36:16.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:16 smithi083 ceph-mon[63709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:36:16.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:16 smithi083 ceph-mon[63709]: osdmap e144: 8 total, 7 up, 8 in 2024-09-18T07:36:16.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:16.941 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:36:17.321 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:16 smithi096 podman[54965]: 2024-09-18 07:36:16.947316357 +0000 UTC m=+0.648974430 container remove a8dc7a20676a87b8bf3df7d43ef513533bc73a8f86ceecee74ddb5d49ca6fed1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vcs-type=git, name=centos-stream, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.buildah.version=1.19.8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870) 2024-09-18T07:36:17.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 240858 9.96 MB/sec execute 506 sec latency 1052.337 ms 2024-09-18T07:36:17.612 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:17 smithi096 podman[55065]: 2024-09-18 07:36:17.221738987 +0000 UTC m=+0.016177107 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:36:17.612 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:17 smithi096 podman[55065]: 2024-09-18 07:36:17.348313512 +0000 UTC m=+0.142751628 container create 243f0eab515979c9a3bc5206382894caa8c08ab3a5d286785dbe8af571a273a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS) 2024-09-18T07:36:17.612 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:17 smithi096 podman[55065]: 2024-09-18 07:36:17.610447225 +0000 UTC m=+0.404885339 container init 243f0eab515979c9a3bc5206382894caa8c08ab3a5d286785dbe8af571a273a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD) 2024-09-18T07:36:17.742 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:17 smithi083 ceph-mon[63709]: osdmap e145: 8 total, 7 up, 8 in 2024-09-18T07:36:17.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:17 smithi096 ceph-mon[49483]: osdmap e145: 8 total, 7 up, 8 in 2024-09-18T07:36:17.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:17 smithi096 podman[55065]: 2024-09-18 07:36:17.61553591 +0000 UTC m=+0.409974023 container start 243f0eab515979c9a3bc5206382894caa8c08ab3a5d286785dbe8af571a273a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-18T07:36:17.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:17 smithi096 podman[55065]: 2024-09-18 07:36:17.818254157 +0000 UTC m=+0.612692267 container attach 243f0eab515979c9a3bc5206382894caa8c08ab3a5d286785dbe8af571a273a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-18T07:36:17.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:17 smithi096 podman[55065]: 2024-09-18 07:36:17.819410828 +0000 UTC m=+0.613848944 container died 243f0eab515979c9a3bc5206382894caa8c08ab3a5d286785dbe8af571a273a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-18T07:36:18.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 241208 9.96 MB/sec execute 507 sec latency 50.178 ms 2024-09-18T07:36:18.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:18 smithi083 ceph-mon[63709]: pgmap v241: 97 pgs: 29 peering, 1 stale+active+clean, 67 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 529 op/s; 0 B/s, 2 objects/s recovering 2024-09-18T07:36:18.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:18 smithi083 ceph-mon[63709]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-18T07:36:18.967 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:18 smithi096 ceph-mon[49483]: pgmap v241: 97 pgs: 29 peering, 1 stale+active+clean, 67 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 529 op/s; 0 B/s, 2 objects/s recovering 2024-09-18T07:36:18.967 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:18 smithi096 ceph-mon[49483]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-18T07:36:18.967 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:18 smithi096 podman[55088]: 2024-09-18 07:36:18.689800796 +0000 UTC m=+0.862251729 container remove 243f0eab515979c9a3bc5206382894caa8c08ab3a5d286785dbe8af571a273a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:36:18.967 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:18 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-18T07:36:18.968 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:18 smithi096 systemd[1]: Stopped Ceph osd.5 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:36:18.968 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:18 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 50.357s CPU time. 2024-09-18T07:36:19.235 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:18 smithi096 systemd[1]: Starting Ceph osd.5 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:36:19.235 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:19 smithi096 podman[55186]: 2024-09-18 07:36:19.105268861 +0000 UTC m=+0.021058121 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:36:19.235 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:19 smithi096 podman[55186]: 2024-09-18 07:36:19.233278527 +0000 UTC m=+0.149067780 container create 4ab21a785f3db551758edfe7b619275458d2cf79da871e4d65076438ae224ae4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5) 2024-09-18T07:36:19.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 241467 9.96 MB/sec execute 508 sec latency 94.752 ms 2024-09-18T07:36:19.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:19 smithi096 podman[55186]: 2024-09-18 07:36:19.5029842 +0000 UTC m=+0.418773454 container init 4ab21a785f3db551758edfe7b619275458d2cf79da871e4d65076438ae224ae4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-18T07:36:19.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:19 smithi096 podman[55186]: 2024-09-18 07:36:19.507454942 +0000 UTC m=+0.423244197 container start 4ab21a785f3db551758edfe7b619275458d2cf79da871e4d65076438ae224ae4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0) 2024-09-18T07:36:19.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:19 smithi096 podman[55186]: 2024-09-18 07:36:19.603114705 +0000 UTC m=+0.518903959 container attach 4ab21a785f3db551758edfe7b619275458d2cf79da871e4d65076438ae224ae4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-18T07:36:19.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:19 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate[55200]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:36:19.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:19 smithi096 bash[55186]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:36:19.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:19 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate[55200]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:36:19.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:19 smithi096 bash[55186]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:36:20.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 241721 9.95 MB/sec execute 509 sec latency 71.804 ms 2024-09-18T07:36:20.846 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:20 smithi083 ceph-mon[63709]: pgmap v242: 97 pgs: 29 peering, 1 stale+active+clean, 67 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.4 MiB/s wr, 298 op/s 2024-09-18T07:36:21.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:20 smithi096 ceph-mon[49483]: pgmap v242: 97 pgs: 29 peering, 1 stale+active+clean, 67 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.4 MiB/s wr, 298 op/s 2024-09-18T07:36:21.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 242003 9.96 MB/sec execute 510 sec latency 59.284 ms 2024-09-18T07:36:21.700 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:36:22.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:22 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:22.334 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:22 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:22.334 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate[55200]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-18T07:36:22.334 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 bash[55186]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-18T07:36:22.334 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate[55200]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-18T07:36:22.335 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 bash[55186]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-18T07:36:22.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 242223 9.95 MB/sec execute 511 sec latency 57.250 ms 2024-09-18T07:36:22.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate[55200]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-18T07:36:22.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 bash[55186]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-18T07:36:22.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate[55200]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-18T07:36:22.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 bash[55186]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-18T07:36:22.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate[55200]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-18T07:36:22.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 bash[55186]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-18T07:36:22.623 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate[55200]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-18T07:36:22.623 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 bash[55186]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-18T07:36:22.623 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate[55200]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-18T07:36:22.623 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 bash[55186]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-18T07:36:22.623 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 podman[55186]: 2024-09-18 07:36:22.369583297 +0000 UTC m=+3.285372548 container died 4ab21a785f3db551758edfe7b619275458d2cf79da871e4d65076438ae224ae4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-18T07:36:22.700 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:36:22.700 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (7m) 73s ago 17m 20.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:36:22.700 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (4m) 73s ago 16m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:36:22.701 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (4m) 33s ago 16m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:36:22.701 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (7m) 73s ago 16m 45.0M - dad864ee21e9 06ff04a18f50 2024-09-18T07:36:22.701 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (11m) 73s ago 11m 41.2M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:36:22.701 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (11m) 33s ago 11m 16.7M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:36:22.701 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (6m) 73s ago 18m 520M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:36:22.701 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (6m) 33s ago 16m 457M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:36:22.701 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (5m) 73s ago 18m 51.7M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:36:22.702 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (5m) 33s ago 16m 41.5M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:36:22.702 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (10m) 73s ago 10m 279M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:36:22.702 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (10m) 33s ago 10m 90.6M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:36:22.702 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (7m) 73s ago 16m 9.79M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:36:22.702 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (7m) 33s ago 16m 11.9M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:36:22.702 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (3m) 73s ago 15m 365M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:36:22.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3m) 73s ago 14m 277M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:36:22.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2m) 73s ago 14m 212M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:36:22.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (84s) 73s ago 14m 11.7M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:36:22.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (38s) 33s ago 13m 14.3M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2ca3dcb5baa0 2024-09-18T07:36:22.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (13m) 33s ago 13m 548M 2704M 17.2.0 e1d6a67b021e a8dc7a20676a 2024-09-18T07:36:22.703 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (12m) 33s ago 12m 474M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:36:22.704 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (12m) 33s ago 12m 759M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:36:22.704 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (6m) 73s ago 15m 53.3M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:36:23.145 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:23 smithi083 ceph-mon[63709]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.4 MiB/s wr, 571 op/s; 181/1653 objects degraded (10.950%) 2024-09-18T07:36:23.145 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:23 smithi083 ceph-mon[63709]: from='client.44301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:23.145 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:23 smithi083 ceph-mon[63709]: Health check failed: Degraded data redundancy: 181/1653 objects degraded (10.950%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:23.145 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:23 smithi083 ceph-mon[63709]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-09-18T07:36:23.246 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:36:23.246 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:36:23.246 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 5 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:36:23.247 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-18T07:36:23.248 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 9 2024-09-18T07:36:23.248 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:36:23.248 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:36:23.353 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:23 smithi096 ceph-mon[49483]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.4 MiB/s wr, 571 op/s; 181/1653 objects degraded (10.950%) 2024-09-18T07:36:23.354 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:23 smithi096 ceph-mon[49483]: from='client.44301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:23.354 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:23 smithi096 ceph-mon[49483]: Health check failed: Degraded data redundancy: 181/1653 objects degraded (10.950%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:23.354 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:23 smithi096 ceph-mon[49483]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-09-18T07:36:23.354 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:22 smithi096 podman[55369]: 2024-09-18 07:36:22.991244362 +0000 UTC m=+0.613439603 container remove 4ab21a785f3db551758edfe7b619275458d2cf79da871e4d65076438ae224ae4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-18T07:36:23.354 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:23 smithi096 podman[55426]: 2024-09-18 07:36:23.111648953 +0000 UTC m=+0.017408766 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:36:23.354 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:23 smithi096 podman[55426]: 2024-09-18 07:36:23.233947594 +0000 UTC m=+0.139707416 container create 8bd5b8b7b6540b104a7dee8b1399d9aaa19e0940301d46407c8f9f9ca50d1a85 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:36:23.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 242495 9.94 MB/sec execute 512 sec latency 48.030 ms 2024-09-18T07:36:23.621 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:23 smithi096 podman[55426]: 2024-09-18 07:36:23.570310748 +0000 UTC m=+0.476070561 container init 8bd5b8b7b6540b104a7dee8b1399d9aaa19e0940301d46407c8f9f9ca50d1a85 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0) 2024-09-18T07:36:23.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:23 smithi096 podman[55426]: 2024-09-18 07:36:23.574109206 +0000 UTC m=+0.479869027 container start 8bd5b8b7b6540b104a7dee8b1399d9aaa19e0940301d46407c8f9f9ca50d1a85 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-18T07:36:23.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:23 smithi096 ceph-osd[55445]: -- 172.21.15.96:0/935357968 <== mon.1 v2:172.21.15.96:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5599e0bf81e0 con 0x5599e0b84000 2024-09-18T07:36:23.694 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:36:23.694 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:36:23.694 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:36:23.694 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:36:23.694 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:36:23.694 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:36:23.694 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:36:23.695 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:36:23.695 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:36:23.695 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "11/23 daemons upgraded", 2024-09-18T07:36:23.695 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:36:23.695 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:36:23.695 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:36:24.069 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:23 smithi096 bash[55426]: 8bd5b8b7b6540b104a7dee8b1399d9aaa19e0940301d46407c8f9f9ca50d1a85 2024-09-18T07:36:24.069 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:23 smithi096 systemd[1]: Started Ceph osd.5 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:36:24.165 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:24 smithi083 ceph-mon[63709]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:24.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:24 smithi083 ceph-mon[63709]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:24.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:24 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/3608430509' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:24.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:24.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:24.166 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:24 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:36:24.231 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 181/1653 objects degraded (10.950%), 23 pgs degraded 2024-09-18T07:36:24.231 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-18T07:36:24.231 INFO:teuthology.orchestra.run.smithi083.stdout: osd.5 (root=default,host=smithi096) is down 2024-09-18T07:36:24.231 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 181/1653 objects degraded (10.950%), 23 pgs degraded 2024-09-18T07:36:24.231 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-18T07:36:24.232 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-18T07:36:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-18T07:36:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-18T07:36:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-18T07:36:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-18T07:36:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-18T07:36:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-18T07:36:24.233 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-18T07:36:24.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:24 smithi096 ceph-mon[49483]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:24 smithi096 ceph-mon[49483]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:24 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/3608430509' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:24 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:36:24.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 242756 9.93 MB/sec execute 513 sec latency 49.005 ms 2024-09-18T07:36:25.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:25 smithi083 ceph-mon[63709]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 384 op/s; 181/1653 objects degraded (10.950%) 2024-09-18T07:36:25.192 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:25 smithi083 ceph-mon[63709]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:25.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:25 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1252563633' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:36:25.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:25 smithi096 ceph-mon[49483]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 384 op/s; 181/1653 objects degraded (10.950%) 2024-09-18T07:36:25.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:25 smithi096 ceph-mon[49483]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:25.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:25 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1252563633' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:36:25.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 243022 9.92 MB/sec execute 514 sec latency 66.240 ms 2024-09-18T07:36:25.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:25 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:36:25.461+0000 7fb7ece00640 -1 Falling back to public interface 2024-09-18T07:36:26.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 243345 9.91 MB/sec execute 515 sec latency 55.733 ms 2024-09-18T07:36:27.233 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:27 smithi083 ceph-mon[63709]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.7 MiB/s wr, 309 op/s; 181/1653 objects degraded (10.950%) 2024-09-18T07:36:27.240 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:27 smithi096 ceph-mon[49483]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.7 MiB/s wr, 309 op/s; 181/1653 objects degraded (10.950%) 2024-09-18T07:36:27.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 243721 9.91 MB/sec execute 516 sec latency 77.063 ms 2024-09-18T07:36:28.502 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:28.503 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:28.503 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:28 smithi083 ceph-mon[63709]: pgmap v246: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.1 MiB/s wr, 398 op/s; 178/1542 objects degraded (11.543%) 2024-09-18T07:36:28.503 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:28.503 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:28.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 244112 9.91 MB/sec execute 517 sec latency 60.164 ms 2024-09-18T07:36:28.768 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:28.768 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:28.768 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:28 smithi096 ceph-mon[49483]: pgmap v246: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.1 MiB/s wr, 398 op/s; 178/1542 objects degraded (11.543%) 2024-09-18T07:36:28.768 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:28.768 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:29.548 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:29 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 178/1542 objects degraded (11.543%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:29.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 244432 9.91 MB/sec execute 518 sec latency 81.604 ms 2024-09-18T07:36:29.771 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:29 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 178/1542 objects degraded (11.543%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:30.567 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:30 smithi083 ceph-mon[63709]: pgmap v247: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 362 op/s; 178/1542 objects degraded (11.543%) 2024-09-18T07:36:30.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 244822 9.91 MB/sec execute 519 sec latency 42.690 ms 2024-09-18T07:36:30.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:30 smithi096 ceph-mon[49483]: pgmap v247: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 362 op/s; 178/1542 objects degraded (11.543%) 2024-09-18T07:36:31.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 245128 9.92 MB/sec execute 520 sec latency 47.387 ms 2024-09-18T07:36:32.138 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:32.139 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:36:32.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:32.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:36:32.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 245440 9.92 MB/sec execute 521 sec latency 48.452 ms 2024-09-18T07:36:33.121 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:36:32.810+0000 7fb7ece00640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-18T07:36:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: pgmap v248: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 540 op/s; 185/1611 objects degraded (11.484%) 2024-09-18T07:36:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:36:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:36:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:33.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:33.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:36:33.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:33.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:33.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:33.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-18T07:36:33.169 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: pgmap v248: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 540 op/s; 185/1611 objects degraded (11.484%) 2024-09-18T07:36:33.169 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:33.169 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:33.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:36:33.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:36:33.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:33.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:33.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:36:33.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:33.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:33.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:33.170 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-18T07:36:33.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 245681 9.92 MB/sec execute 522 sec latency 65.082 ms 2024-09-18T07:36:34.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:33 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-18T07:36:34.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:33 smithi083 ceph-mon[63709]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-18T07:36:34.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:33 smithi083 ceph-mon[63709]: from='osd.5 [v2:172.21.15.96:6808/2672972138,v1:172.21.15.96:6809/2672972138]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-18T07:36:34.190 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:33 smithi083 ceph-mon[63709]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-18T07:36:34.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:33 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-18T07:36:34.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:33 smithi096 ceph-mon[49483]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-18T07:36:34.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:33 smithi096 ceph-mon[49483]: from='osd.5 [v2:172.21.15.96:6808/2672972138,v1:172.21.15.96:6809/2672972138]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-18T07:36:34.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:33 smithi096 ceph-mon[49483]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-18T07:36:34.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:33 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:36:33.885+0000 7fb7ece00640 -1 osd.5 143 log_to_monitors true 2024-09-18T07:36:34.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 245886 9.90 MB/sec execute 523 sec latency 99.149 ms 2024-09-18T07:36:34.871 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:36:34 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:36:34.453+0000 7fb7e0ba2640 -1 osd.5 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:36:35.215 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:34 smithi083 ceph-mon[63709]: pgmap v249: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 357 op/s; 185/1611 objects degraded (11.484%) 2024-09-18T07:36:35.216 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:34 smithi083 ceph-mon[63709]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-18T07:36:35.216 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:34 smithi083 ceph-mon[63709]: osdmap e146: 8 total, 7 up, 8 in 2024-09-18T07:36:35.216 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:34 smithi083 ceph-mon[63709]: from='osd.5 [v2:172.21.15.96:6808/2672972138,v1:172.21.15.96:6809/2672972138]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:36:35.216 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:34 smithi083 ceph-mon[63709]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:36:35.216 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:34 smithi083 ceph-mon[63709]: from='osd.5 ' entity='osd.5' 2024-09-18T07:36:35.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:34 smithi096 ceph-mon[49483]: pgmap v249: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 357 op/s; 185/1611 objects degraded (11.484%) 2024-09-18T07:36:35.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:34 smithi096 ceph-mon[49483]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-18T07:36:35.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:34 smithi096 ceph-mon[49483]: osdmap e146: 8 total, 7 up, 8 in 2024-09-18T07:36:35.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:34 smithi096 ceph-mon[49483]: from='osd.5 [v2:172.21.15.96:6808/2672972138,v1:172.21.15.96:6809/2672972138]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:36:35.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:34 smithi096 ceph-mon[49483]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:36:35.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:34 smithi096 ceph-mon[49483]: from='osd.5 ' entity='osd.5' 2024-09-18T07:36:35.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 246127 9.90 MB/sec execute 524 sec latency 79.280 ms 2024-09-18T07:36:35.980 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:35 smithi083 ceph-mon[63709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:36:35.981 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:35 smithi083 ceph-mon[63709]: osd.5 [v2:172.21.15.96:6808/2672972138,v1:172.21.15.96:6809/2672972138] boot 2024-09-18T07:36:35.981 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:35 smithi083 ceph-mon[63709]: osdmap e147: 8 total, 8 up, 8 in 2024-09-18T07:36:35.981 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:35 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-18T07:36:36.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:35 smithi096 ceph-mon[49483]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:36:36.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:35 smithi096 ceph-mon[49483]: osd.5 [v2:172.21.15.96:6808/2672972138,v1:172.21.15.96:6809/2672972138] boot 2024-09-18T07:36:36.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:35 smithi096 ceph-mon[49483]: osdmap e147: 8 total, 8 up, 8 in 2024-09-18T07:36:36.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:35 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-18T07:36:36.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 246283 9.88 MB/sec execute 525 sec latency 1046.917 ms 2024-09-18T07:36:37.014 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:36 smithi083 ceph-mon[63709]: pgmap v252: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.6 MiB/s wr, 430 op/s; 185/1611 objects degraded (11.484%) 2024-09-18T07:36:37.014 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:36 smithi083 ceph-mon[63709]: osdmap e148: 8 total, 8 up, 8 in 2024-09-18T07:36:37.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:36 smithi096 ceph-mon[49483]: pgmap v252: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.6 MiB/s wr, 430 op/s; 185/1611 objects degraded (11.484%) 2024-09-18T07:36:37.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:36 smithi096 ceph-mon[49483]: osdmap e148: 8 total, 8 up, 8 in 2024-09-18T07:36:37.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 246510 9.88 MB/sec execute 526 sec latency 2018.148 ms 2024-09-18T07:36:38.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 247063 9.87 MB/sec execute 527 sec latency 104.520 ms 2024-09-18T07:36:38.717 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:38 smithi083 ceph-mon[63709]: osdmap e149: 8 total, 8 up, 8 in 2024-09-18T07:36:38.717 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:38 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 185/1611 objects degraded (11.484%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:38.718 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:38 smithi083 ceph-mon[63709]: pgmap v255: 97 pgs: 2 remapped+peering, 10 peering, 3 active+recovery_wait+degraded, 3 active+undersized, 1 active+recovering, 11 active+undersized+degraded, 67 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 453 op/s; 123/1632 objects degraded (7.537%); 23 KiB/s, 0 objects/s recovering 2024-09-18T07:36:38.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:38 smithi096 ceph-mon[49483]: osdmap e149: 8 total, 8 up, 8 in 2024-09-18T07:36:38.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:38 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 185/1611 objects degraded (11.484%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:38.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:38 smithi096 ceph-mon[49483]: pgmap v255: 97 pgs: 2 remapped+peering, 10 peering, 3 active+recovery_wait+degraded, 3 active+undersized, 1 active+recovering, 11 active+undersized+degraded, 67 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 453 op/s; 123/1632 objects degraded (7.537%); 23 KiB/s, 0 objects/s recovering 2024-09-18T07:36:39.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 247198 9.86 MB/sec execute 528 sec latency 1001.067 ms 2024-09-18T07:36:39.719 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:39 smithi083 ceph-mon[63709]: osdmap e150: 8 total, 8 up, 8 in 2024-09-18T07:36:39.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:39 smithi096 ceph-mon[49483]: osdmap e150: 8 total, 8 up, 8 in 2024-09-18T07:36:40.485 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:40 smithi083 ceph-mon[63709]: osdmap e151: 8 total, 8 up, 8 in 2024-09-18T07:36:40.486 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:40 smithi083 ceph-mon[63709]: pgmap v258: 97 pgs: 2 remapped+peering, 13 peering, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+degraded, 77 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.8 MiB/s wr, 164 op/s; 36/1599 objects degraded (2.251%); 497 KiB/s, 25 objects/s recovering 2024-09-18T07:36:40.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 247669 9.87 MB/sec execute 529 sec latency 28.462 ms 2024-09-18T07:36:40.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:40 smithi096 ceph-mon[49483]: osdmap e151: 8 total, 8 up, 8 in 2024-09-18T07:36:40.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:40 smithi096 ceph-mon[49483]: pgmap v258: 97 pgs: 2 remapped+peering, 13 peering, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+degraded, 77 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.8 MiB/s wr, 164 op/s; 36/1599 objects degraded (2.251%); 497 KiB/s, 25 objects/s recovering 2024-09-18T07:36:41.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 248089 9.87 MB/sec execute 530 sec latency 39.633 ms 2024-09-18T07:36:41.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:41 smithi083 ceph-mon[63709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/1599 objects degraded (2.251%), 4 pgs degraded) 2024-09-18T07:36:41.751 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:41 smithi083 ceph-mon[63709]: Cluster is now healthy 2024-09-18T07:36:42.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:41 smithi096 ceph-mon[49483]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/1599 objects degraded (2.251%), 4 pgs degraded) 2024-09-18T07:36:42.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:41 smithi096 ceph-mon[49483]: Cluster is now healthy 2024-09-18T07:36:42.605 INFO:teuthology.orchestra.run.smithi083.stdout: 5 248486 9.88 MB/sec execute 531 sec latency 50.036 ms 2024-09-18T07:36:42.804 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:42 smithi083 ceph-mon[63709]: pgmap v259: 97 pgs: 3 peering, 94 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 5.3 MiB/s wr, 564 op/s; 2.2 MiB/s, 46 objects/s recovering 2024-09-18T07:36:43.032 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:42 smithi096 ceph-mon[49483]: pgmap v259: 97 pgs: 3 peering, 94 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 5.3 MiB/s wr, 564 op/s; 2.2 MiB/s, 46 objects/s recovering 2024-09-18T07:36:43.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 248740 9.88 MB/sec execute 532 sec latency 46.378 ms 2024-09-18T07:36:44.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 249097 9.88 MB/sec execute 533 sec latency 42.901 ms 2024-09-18T07:36:44.861 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:44 smithi083 ceph-mon[63709]: pgmap v260: 97 pgs: 3 peering, 94 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 449 op/s; 1.8 MiB/s, 38 objects/s recovering 2024-09-18T07:36:45.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:44 smithi096 ceph-mon[49483]: pgmap v260: 97 pgs: 3 peering, 94 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 449 op/s; 1.8 MiB/s, 38 objects/s recovering 2024-09-18T07:36:45.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 249401 9.88 MB/sec execute 534 sec latency 46.465 ms 2024-09-18T07:36:46.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 249658 9.87 MB/sec execute 535 sec latency 50.112 ms 2024-09-18T07:36:46.953 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:46 smithi083 ceph-mon[63709]: pgmap v261: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.3 MiB/s wr, 384 op/s; 1.4 MiB/s, 29 objects/s recovering 2024-09-18T07:36:46.954 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:46.954 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:36:47.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:46 smithi096 ceph-mon[49483]: pgmap v261: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.3 MiB/s wr, 384 op/s; 1.4 MiB/s, 29 objects/s recovering 2024-09-18T07:36:47.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:47.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:36:47.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 249894 9.87 MB/sec execute 536 sec latency 57.497 ms 2024-09-18T07:36:48.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250190 9.86 MB/sec execute 537 sec latency 66.569 ms 2024-09-18T07:36:48.772 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:48 smithi083 ceph-mon[63709]: pgmap v262: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 1.2 MiB/s, 25 objects/s recovering 2024-09-18T07:36:48.772 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:48 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-18T07:36:48.772 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:48 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-18T07:36:48.772 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:48 smithi083 ceph-mon[63709]: Upgrade: osd.6 is safe to restart 2024-09-18T07:36:48.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:48 smithi096 ceph-mon[49483]: pgmap v262: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 1.2 MiB/s, 25 objects/s recovering 2024-09-18T07:36:48.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:48 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-18T07:36:48.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:48 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-18T07:36:48.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:48 smithi096 ceph-mon[49483]: Upgrade: osd.6 is safe to restart 2024-09-18T07:36:49.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250481 9.85 MB/sec execute 538 sec latency 67.383 ms 2024-09-18T07:36:50.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 250745 9.84 MB/sec execute 539 sec latency 63.564 ms 2024-09-18T07:36:50.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:50 smithi083 ceph-mon[63709]: Upgrade: Updating osd.6 2024-09-18T07:36:50.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:50.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-18T07:36:50.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:36:50.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:50 smithi083 ceph-mon[63709]: Deploying daemon osd.6 on smithi096 2024-09-18T07:36:50.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:50 smithi083 ceph-mon[63709]: pgmap v263: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 472 op/s; 927 KiB/s, 13 objects/s recovering 2024-09-18T07:36:50.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:50 smithi096 ceph-mon[49483]: Upgrade: Updating osd.6 2024-09-18T07:36:50.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:50.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-18T07:36:50.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:36:50.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:50 smithi096 ceph-mon[49483]: Deploying daemon osd.6 on smithi096 2024-09-18T07:36:50.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:50 smithi096 ceph-mon[49483]: pgmap v263: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 472 op/s; 927 KiB/s, 13 objects/s recovering 2024-09-18T07:36:51.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 251033 9.83 MB/sec execute 540 sec latency 54.037 ms 2024-09-18T07:36:52.186 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:52.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:36:52.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 251413 9.84 MB/sec execute 541 sec latency 60.624 ms 2024-09-18T07:36:53.196 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:53 smithi083 ceph-mon[63709]: pgmap v264: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 609 op/s; 783 KiB/s, 11 objects/s recovering 2024-09-18T07:36:53.238 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:53 smithi096 ceph-mon[49483]: pgmap v264: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 609 op/s; 783 KiB/s, 11 objects/s recovering 2024-09-18T07:36:53.239 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:52 smithi096 systemd[1]: Stopping Ceph osd.6 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:36:53.606 INFO:teuthology.orchestra.run.smithi083.stdout: 5 251868 9.84 MB/sec execute 542 sec latency 51.092 ms 2024-09-18T07:36:53.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:53 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[36041]: 2024-09-18T07:36:53.234+0000 7f5ebd791700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:36:53.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:53 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[36041]: 2024-09-18T07:36:53.234+0000 7f5ebd791700 -1 osd.6 151 *** Got signal Terminated *** 2024-09-18T07:36:53.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:53 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[36041]: 2024-09-18T07:36:53.234+0000 7f5ebd791700 -1 osd.6 151 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:36:54.220 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:54 smithi083 ceph-mon[63709]: osd.6 marked itself down and dead 2024-09-18T07:36:54.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:54 smithi096 ceph-mon[49483]: osd.6 marked itself down and dead 2024-09-18T07:36:54.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 252112 9.84 MB/sec execute 543 sec latency 506.669 ms 2024-09-18T07:36:54.742 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:36:55.038 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:54 smithi096 podman[57007]: 2024-09-18 07:36:54.728100648 +0000 UTC m=+1.623084923 container died 6053224db870bd2dbe4a81cef89e3d726dd5f1157b57fb2c094a1921fd2de252 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, architecture=x86_64, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, ceph=True, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-type=git, release=754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-18T07:36:55.334 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:55 smithi083 ceph-mon[63709]: pgmap v265: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 392 op/s 2024-09-18T07:36:55.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:55 smithi083 ceph-mon[63709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:36:55.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:55 smithi083 ceph-mon[63709]: osdmap e152: 8 total, 7 up, 8 in 2024-09-18T07:36:55.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:55 smithi096 ceph-mon[49483]: pgmap v265: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 392 op/s 2024-09-18T07:36:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:55 smithi096 ceph-mon[49483]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:36:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:55 smithi096 ceph-mon[49483]: osdmap e152: 8 total, 7 up, 8 in 2024-09-18T07:36:55.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:55 smithi096 podman[57007]: 2024-09-18 07:36:55.227939821 +0000 UTC m=+2.122924100 container cleanup 6053224db870bd2dbe4a81cef89e3d726dd5f1157b57fb2c094a1921fd2de252 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6, version=8, architecture=x86_64, distribution-scope=public, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, vcs-type=git, release=754, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc.) 2024-09-18T07:36:55.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:55 smithi096 bash[57007]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6 2024-09-18T07:36:55.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 252333 9.83 MB/sec execute 544 sec latency 1028.921 ms 2024-09-18T07:36:55.726 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:36:55.726 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (8m) 106s ago 17m 20.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (5m) 106s ago 17m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (5m) 28s ago 17m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (8m) 106s ago 17m 45.0M - dad864ee21e9 06ff04a18f50 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (11m) 106s ago 11m 41.2M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (11m) 28s ago 11m 16.8M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (7m) 106s ago 18m 520M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (6m) 28s ago 16m 457M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (6m) 106s ago 19m 51.7M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (5m) 28s ago 16m 44.1M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (11m) 106s ago 11m 279M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (11m) 28s ago 11m 90.6M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (8m) 106s ago 17m 9.79M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (7m) 28s ago 16m 11.2M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:36:55.727 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (4m) 106s ago 16m 365M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:36:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (3m) 106s ago 15m 277M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:36:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (2m) 106s ago 15m 212M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:36:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (117s) 106s ago 14m 11.7M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:36:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (71s) 28s ago 14m 273M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2ca3dcb5baa0 2024-09-18T07:36:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (32s) 28s ago 13m 12.1M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 8bd5b8b7b654 2024-09-18T07:36:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (13m) 28s ago 13m 503M 2704M 17.2.0 e1d6a67b021e 6053224db870 2024-09-18T07:36:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (13m) 28s ago 13m 560M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:36:55.728 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (6m) 106s ago 16m 53.3M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:36:55.871 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:55 smithi096 podman[57021]: 2024-09-18 07:36:55.504795878 +0000 UTC m=+0.773747408 container remove 6053224db870bd2dbe4a81cef89e3d726dd5f1157b57fb2c094a1921fd2de252 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, name=centos-stream, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container) 2024-09-18T07:36:56.211 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:56 smithi096 ceph-mon[49483]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:56.211 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:56 smithi096 ceph-mon[49483]: osdmap e153: 8 total, 7 up, 8 in 2024-09-18T07:36:56.211 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:55 smithi096 podman[57106]: 2024-09-18 07:36:55.81958457 +0000 UTC m=+0.023084225 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:36:56.211 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:55 smithi096 podman[57106]: 2024-09-18 07:36:55.956048788 +0000 UTC m=+0.159548421 container create 11290ae39adc24a68b03bcc592b4eb887ffa620be65e394abc61b27f5236f45f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-09-18T07:36:56.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:56 smithi083 ceph-mon[63709]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:56.357 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:56 smithi083 ceph-mon[63709]: osdmap e153: 8 total, 7 up, 8 in 2024-09-18T07:36:56.475 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:36:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:36:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:36:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:36:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:36:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:36:56.475 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 6 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 10 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:36:56.476 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:36:56.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 252884 9.85 MB/sec execute 545 sec latency 37.549 ms 2024-09-18T07:36:56.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:56 smithi096 podman[57106]: 2024-09-18 07:36:56.209020434 +0000 UTC m=+0.412520070 container init 11290ae39adc24a68b03bcc592b4eb887ffa620be65e394abc61b27f5236f45f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-18T07:36:56.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:56 smithi096 podman[57106]: 2024-09-18 07:36:56.214809976 +0000 UTC m=+0.418309610 container start 11290ae39adc24a68b03bcc592b4eb887ffa620be65e394abc61b27f5236f45f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:36:56.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:56 smithi096 podman[57106]: 2024-09-18 07:36:56.317480654 +0000 UTC m=+0.520980284 container attach 11290ae39adc24a68b03bcc592b4eb887ffa620be65e394abc61b27f5236f45f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-18T07:36:56.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:56 smithi096 podman[57106]: 2024-09-18 07:36:56.455714376 +0000 UTC m=+0.659214011 container died 11290ae39adc24a68b03bcc592b4eb887ffa620be65e394abc61b27f5236f45f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-18T07:36:57.058 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:36:57.058 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:36:57.058 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "12/23 daemons upgraded", 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:36:57.059 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:36:57.362 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:57 smithi083 ceph-mon[63709]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:57.363 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:57 smithi083 ceph-mon[63709]: pgmap v268: 97 pgs: 10 stale+active+clean, 87 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.7 MiB/s wr, 341 op/s 2024-09-18T07:36:57.363 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:57 smithi083 ceph-mon[63709]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:57.363 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:57 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/2143575007' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:57.363 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:57 smithi083 ceph-mon[63709]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:57.549 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:57 smithi096 ceph-mon[49483]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:57.550 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:57 smithi096 ceph-mon[49483]: pgmap v268: 97 pgs: 10 stale+active+clean, 87 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.7 MiB/s wr, 341 op/s 2024-09-18T07:36:57.550 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:57 smithi096 ceph-mon[49483]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:57.550 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:57 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/2143575007' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:36:57.550 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:57 smithi096 ceph-mon[49483]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:36:57.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 253423 9.86 MB/sec execute 546 sec latency 32.233 ms 2024-09-18T07:36:57.717 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN 1 osds down 2024-09-18T07:36:57.717 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-18T07:36:57.717 INFO:teuthology.orchestra.run.smithi083.stdout: osd.6 (root=default,host=smithi096) is down 2024-09-18T07:36:57.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:57 smithi096 podman[57129]: 2024-09-18 07:36:57.547640158 +0000 UTC m=+1.084594381 container remove 11290ae39adc24a68b03bcc592b4eb887ffa620be65e394abc61b27f5236f45f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-18T07:36:57.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:57 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-18T07:36:57.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:57 smithi096 systemd[1]: Stopped Ceph osd.6 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:36:57.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:57 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.6.service: Consumed 4min 33.080s CPU time. 2024-09-18T07:36:58.301 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:58 smithi096 ceph-mon[49483]: pgmap v269: 97 pgs: 8 active+undersized, 2 stale+active+clean, 20 active+undersized+degraded, 67 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.5 MiB/s wr, 602 op/s; 164/1650 objects degraded (9.939%) 2024-09-18T07:36:58.301 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:58 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/697704849' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:36:58.301 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:57 smithi096 systemd[1]: Starting Ceph osd.6 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:36:58.301 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:58 smithi096 podman[57242]: 2024-09-18 07:36:58.100845315 +0000 UTC m=+0.020818195 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:36:58.373 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:58 smithi083 ceph-mon[63709]: pgmap v269: 97 pgs: 8 active+undersized, 2 stale+active+clean, 20 active+undersized+degraded, 67 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.5 MiB/s wr, 602 op/s; 164/1650 objects degraded (9.939%) 2024-09-18T07:36:58.373 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:58 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/697704849' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:36:58.570 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:58 smithi096 podman[57242]: 2024-09-18 07:36:58.299290287 +0000 UTC m=+0.219263172 container create fa4f4868f2b5a60692d6907b6c0367a8e9f24d188c31bad79454d0b0441afab0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-18T07:36:58.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 253689 9.85 MB/sec execute 547 sec latency 48.675 ms 2024-09-18T07:36:58.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:58 smithi096 podman[57242]: 2024-09-18 07:36:58.56881042 +0000 UTC m=+0.488783299 container init fa4f4868f2b5a60692d6907b6c0367a8e9f24d188c31bad79454d0b0441afab0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:36:58.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:58 smithi096 podman[57242]: 2024-09-18 07:36:58.572509895 +0000 UTC m=+0.492482779 container start fa4f4868f2b5a60692d6907b6c0367a8e9f24d188c31bad79454d0b0441afab0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True) 2024-09-18T07:36:58.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:58 smithi096 podman[57242]: 2024-09-18 07:36:58.66919006 +0000 UTC m=+0.589162949 container attach fa4f4868f2b5a60692d6907b6c0367a8e9f24d188c31bad79454d0b0441afab0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5) 2024-09-18T07:36:58.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate[57257]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:36:58.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:58 smithi096 bash[57242]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:36:58.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate[57257]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:36:58.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:36:58 smithi096 bash[57242]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:36:59.154 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:36:59 smithi083 ceph-mon[63709]: Health check failed: Degraded data redundancy: 164/1650 objects degraded (9.939%), 20 pgs degraded (PG_DEGRADED) 2024-09-18T07:36:59.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 254069 9.85 MB/sec execute 548 sec latency 58.031 ms 2024-09-18T07:36:59.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:36:59 smithi096 ceph-mon[49483]: Health check failed: Degraded data redundancy: 164/1650 objects degraded (9.939%), 20 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:00.212 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:00 smithi083 ceph-mon[63709]: pgmap v270: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 370 op/s; 199/1659 objects degraded (11.995%) 2024-09-18T07:37:00.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 254451 9.84 MB/sec execute 549 sec latency 55.679 ms 2024-09-18T07:37:00.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:00 smithi096 ceph-mon[49483]: pgmap v270: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 370 op/s; 199/1659 objects degraded (11.995%) 2024-09-18T07:37:01.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate[57257]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-18T07:37:01.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 bash[57242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-18T07:37:01.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate[57257]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-18T07:37:01.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 bash[57242]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-18T07:37:01.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 254839 9.85 MB/sec execute 550 sec latency 50.429 ms 2024-09-18T07:37:01.750 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate[57257]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-18T07:37:01.750 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 bash[57242]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-18T07:37:01.750 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate[57257]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-18T07:37:01.750 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 bash[57242]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-18T07:37:01.751 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate[57257]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-18T07:37:01.751 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 bash[57242]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-18T07:37:01.751 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate[57257]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-18T07:37:01.751 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 bash[57242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-18T07:37:01.751 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate[57257]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-18T07:37:01.751 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 bash[57242]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-18T07:37:01.751 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:01 smithi096 podman[57242]: 2024-09-18 07:37:01.430776817 +0000 UTC m=+3.350749698 container died fa4f4868f2b5a60692d6907b6c0367a8e9f24d188c31bad79454d0b0441afab0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-18T07:37:02.015 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:02.016 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:37:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:37:02.464 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:02 smithi096 podman[57405]: 2024-09-18 07:37:02.185303981 +0000 UTC m=+0.748237615 container remove fa4f4868f2b5a60692d6907b6c0367a8e9f24d188c31bad79454d0b0441afab0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:37:02.464 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:02 smithi096 podman[57463]: 2024-09-18 07:37:02.30870323 +0000 UTC m=+0.019427554 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:37:02.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 255274 9.85 MB/sec execute 551 sec latency 37.513 ms 2024-09-18T07:37:02.736 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:02 smithi096 podman[57463]: 2024-09-18 07:37:02.462454456 +0000 UTC m=+0.173178762 container create c3e35aae405ecbae9d1d3ee3f491c3c1bcb3bba0fd75b2a9bea70b7c4c0af25b (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-18T07:37:03.027 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:02 smithi083 ceph-mon[63709]: pgmap v271: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 6.6 MiB/s wr, 587 op/s; 180/1611 objects degraded (11.173%) 2024-09-18T07:37:03.028 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:02 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:03.028 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:02 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:03.028 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:02 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:37:03.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:02 smithi096 ceph-mon[49483]: pgmap v271: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 6.6 MiB/s wr, 587 op/s; 180/1611 objects degraded (11.173%) 2024-09-18T07:37:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:02 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:02 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:02 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:37:03.122 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:02 smithi096 podman[57463]: 2024-09-18 07:37:02.734204968 +0000 UTC m=+0.444929279 container init c3e35aae405ecbae9d1d3ee3f491c3c1bcb3bba0fd75b2a9bea70b7c4c0af25b (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-18T07:37:03.122 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:02 smithi096 podman[57463]: 2024-09-18 07:37:02.738264567 +0000 UTC m=+0.448988877 container start c3e35aae405ecbae9d1d3ee3f491c3c1bcb3bba0fd75b2a9bea70b7c4c0af25b (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-18T07:37:03.122 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:02 smithi096 ceph-osd[57482]: -- 172.21.15.96:0/2300838123 <== mon.1 v2:172.21.15.96:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ff7587c1e0 con 0x55ff75808000 2024-09-18T07:37:03.122 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:02 smithi096 bash[57463]: c3e35aae405ecbae9d1d3ee3f491c3c1bcb3bba0fd75b2a9bea70b7c4c0af25b 2024-09-18T07:37:03.122 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:02 smithi096 systemd[1]: Started Ceph osd.6 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:37:03.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 255693 9.85 MB/sec execute 552 sec latency 40.350 ms 2024-09-18T07:37:04.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 256050 9.86 MB/sec execute 553 sec latency 47.141 ms 2024-09-18T07:37:04.621 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:04 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:37:04.367+0000 7fceff2ad640 -1 Falling back to public interface 2024-09-18T07:37:05.057 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:04 smithi083 ceph-mon[63709]: pgmap v272: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.4 MiB/s wr, 478 op/s; 180/1611 objects degraded (11.173%) 2024-09-18T07:37:05.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:04 smithi096 ceph-mon[49483]: pgmap v272: 97 pgs: 11 active+undersized, 24 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 5.4 MiB/s wr, 478 op/s; 180/1611 objects degraded (11.173%) 2024-09-18T07:37:05.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 256377 9.86 MB/sec execute 554 sec latency 52.006 ms 2024-09-18T07:37:06.085 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:06 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 180/1614 objects degraded (11.152%), 25 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:06.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:06 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 180/1614 objects degraded (11.152%), 25 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:06.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 256724 9.86 MB/sec execute 555 sec latency 82.113 ms 2024-09-18T07:37:07.109 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:07 smithi083 ceph-mon[63709]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.8 MiB/s wr, 606 op/s; 180/1614 objects degraded (11.152%) 2024-09-18T07:37:07.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:07 smithi096 ceph-mon[49483]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.8 MiB/s wr, 606 op/s; 180/1614 objects degraded (11.152%) 2024-09-18T07:37:07.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 257105 9.86 MB/sec execute 556 sec latency 42.288 ms 2024-09-18T07:37:08.140 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:08.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:08.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:08 smithi083 ceph-mon[63709]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.1 MiB/s wr, 686 op/s; 193/1647 objects degraded (11.718%) 2024-09-18T07:37:08.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:08.141 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:08.375 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:08.376 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:08.376 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:08 smithi096 ceph-mon[49483]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.1 MiB/s wr, 686 op/s; 193/1647 objects degraded (11.718%) 2024-09-18T07:37:08.376 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:08.376 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:08.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 257357 9.85 MB/sec execute 557 sec latency 37.345 ms 2024-09-18T07:37:09.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 257779 9.85 MB/sec execute 558 sec latency 50.536 ms 2024-09-18T07:37:10.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 258261 9.85 MB/sec execute 559 sec latency 37.622 ms 2024-09-18T07:37:10.784 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:10 smithi083 ceph-mon[63709]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 587 op/s; 196/1641 objects degraded (11.944%) 2024-09-18T07:37:10.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:10 smithi096 ceph-mon[49483]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 587 op/s; 196/1641 objects degraded (11.944%) 2024-09-18T07:37:11.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 258656 9.86 MB/sec execute 560 sec latency 50.779 ms 2024-09-18T07:37:11.871 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:11 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:37:11.529+0000 7fceff2ad640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.8 MiB/s wr, 658 op/s; 202/1572 objects degraded (12.850%) 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:12.307 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:12.308 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:12.308 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-18T07:37:12.308 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-18T07:37:12.308 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:12 smithi083 ceph-mon[63709]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.8 MiB/s wr, 658 op/s; 202/1572 objects degraded (12.850%) 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:12.604 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:12.605 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:12.605 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-18T07:37:12.605 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-18T07:37:12.605 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-mon[49483]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-18T07:37:12.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 259168 9.86 MB/sec execute 561 sec latency 28.272 ms 2024-09-18T07:37:12.871 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:12 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:37:12.663+0000 7fceff2ad640 -1 osd.6 151 log_to_monitors true 2024-09-18T07:37:13.316 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:13 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 202/1572 objects degraded (12.850%), 25 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:13.317 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:13 smithi083 ceph-mon[63709]: from='osd.6 [v2:172.21.15.96:6816/1962694094,v1:172.21.15.96:6817/1962694094]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-18T07:37:13.317 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:13 smithi083 ceph-mon[63709]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-18T07:37:13.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 259477 9.86 MB/sec execute 562 sec latency 50.015 ms 2024-09-18T07:37:13.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:13 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 202/1572 objects degraded (12.850%), 25 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:13.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:13 smithi096 ceph-mon[49483]: from='osd.6 [v2:172.21.15.96:6816/1962694094,v1:172.21.15.96:6817/1962694094]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-18T07:37:13.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:13 smithi096 ceph-mon[49483]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-18T07:37:13.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:37:13 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:37:13.310+0000 7fcef7057640 -1 osd.6 151 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:37:14.341 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:14 smithi083 ceph-mon[63709]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-18T07:37:14.342 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:14 smithi083 ceph-mon[63709]: osdmap e154: 8 total, 7 up, 8 in 2024-09-18T07:37:14.342 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:14 smithi083 ceph-mon[63709]: from='osd.6 [v2:172.21.15.96:6816/1962694094,v1:172.21.15.96:6817/1962694094]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:37:14.342 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:14 smithi083 ceph-mon[63709]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:37:14.342 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:14 smithi083 ceph-mon[63709]: pgmap v278: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.8 MiB/s wr, 616 op/s; 202/1572 objects degraded (12.850%) 2024-09-18T07:37:14.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 259794 9.87 MB/sec execute 563 sec latency 290.918 ms 2024-09-18T07:37:14.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:14 smithi096 ceph-mon[49483]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-18T07:37:14.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:14 smithi096 ceph-mon[49483]: osdmap e154: 8 total, 7 up, 8 in 2024-09-18T07:37:14.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:14 smithi096 ceph-mon[49483]: from='osd.6 [v2:172.21.15.96:6816/1962694094,v1:172.21.15.96:6817/1962694094]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:37:14.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:14 smithi096 ceph-mon[49483]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:37:14.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:14 smithi096 ceph-mon[49483]: pgmap v278: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.8 MiB/s wr, 616 op/s; 202/1572 objects degraded (12.850%) 2024-09-18T07:37:15.500 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:15 smithi083 ceph-mon[63709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:37:15.500 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:15 smithi083 ceph-mon[63709]: osd.6 [v2:172.21.15.96:6816/1962694094,v1:172.21.15.96:6817/1962694094] boot 2024-09-18T07:37:15.500 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:15 smithi083 ceph-mon[63709]: osdmap e155: 8 total, 8 up, 8 in 2024-09-18T07:37:15.501 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:15 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-18T07:37:15.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 259828 9.85 MB/sec execute 564 sec latency 1121.048 ms 2024-09-18T07:37:15.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:15 smithi096 ceph-mon[49483]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:37:15.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:15 smithi096 ceph-mon[49483]: osd.6 [v2:172.21.15.96:6816/1962694094,v1:172.21.15.96:6817/1962694094] boot 2024-09-18T07:37:15.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:15 smithi096 ceph-mon[49483]: osdmap e155: 8 total, 8 up, 8 in 2024-09-18T07:37:15.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:15 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-18T07:37:16.586 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:16 smithi083 ceph-mon[63709]: osdmap e156: 8 total, 8 up, 8 in 2024-09-18T07:37:16.587 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:16 smithi083 ceph-mon[63709]: pgmap v281: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.6 MiB/s wr, 305 op/s; 199/1578 objects degraded (12.611%) 2024-09-18T07:37:16.587 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:16.587 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:37:16.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 259875 9.83 MB/sec execute 565 sec latency 2118.653 ms 2024-09-18T07:37:16.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:16 smithi096 ceph-mon[49483]: osdmap e156: 8 total, 8 up, 8 in 2024-09-18T07:37:16.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:16 smithi096 ceph-mon[49483]: pgmap v281: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.6 MiB/s wr, 305 op/s; 199/1578 objects degraded (12.611%) 2024-09-18T07:37:16.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:16.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:37:17.377 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:17 smithi083 ceph-mon[63709]: osdmap e157: 8 total, 8 up, 8 in 2024-09-18T07:37:17.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 260224 9.83 MB/sec execute 566 sec latency 1172.539 ms 2024-09-18T07:37:17.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:17 smithi096 ceph-mon[49483]: osdmap e157: 8 total, 8 up, 8 in 2024-09-18T07:37:18.560 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:18 smithi083 ceph-mon[63709]: osdmap e158: 8 total, 8 up, 8 in 2024-09-18T07:37:18.560 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:18 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 199/1578 objects degraded (12.611%), 25 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:18.560 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:18 smithi083 ceph-mon[63709]: pgmap v284: 97 pgs: 2 activating+undersized+degraded+remapped, 7 remapped+peering, 2 active+undersized, 11 active+undersized+degraded, 75 active+clean; 343 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 590 op/s; 112/1647 objects degraded (6.800%); 17 KiB/s, 76 keys/s, 4 objects/s recovering 2024-09-18T07:37:18.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 260342 9.82 MB/sec execute 567 sec latency 991.142 ms 2024-09-18T07:37:18.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:18 smithi096 ceph-mon[49483]: osdmap e158: 8 total, 8 up, 8 in 2024-09-18T07:37:18.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:18 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 199/1578 objects degraded (12.611%), 25 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:18.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:18 smithi096 ceph-mon[49483]: pgmap v284: 97 pgs: 2 activating+undersized+degraded+remapped, 7 remapped+peering, 2 active+undersized, 11 active+undersized+degraded, 75 active+clean; 343 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 590 op/s; 112/1647 objects degraded (6.800%); 17 KiB/s, 76 keys/s, 4 objects/s recovering 2024-09-18T07:37:19.330 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:19 smithi083 ceph-mon[63709]: osdmap e159: 8 total, 8 up, 8 in 2024-09-18T07:37:19.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 260656 9.82 MB/sec execute 568 sec latency 44.614 ms 2024-09-18T07:37:19.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:19 smithi096 ceph-mon[49483]: osdmap e159: 8 total, 8 up, 8 in 2024-09-18T07:37:20.362 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:20 smithi083 ceph-mon[63709]: pgmap v286: 97 pgs: 2 peering, 2 activating+undersized+degraded+remapped, 5 remapped+peering, 88 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.5 MiB/s wr, 495 op/s; 58/1644 objects degraded (3.528%); 2.3 MiB/s, 70 keys/s, 27 objects/s recovering 2024-09-18T07:37:20.617 INFO:teuthology.orchestra.run.smithi083.stdout: 5 260931 9.81 MB/sec execute 569 sec latency 51.401 ms 2024-09-18T07:37:20.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:20 smithi096 ceph-mon[49483]: pgmap v286: 97 pgs: 2 peering, 2 activating+undersized+degraded+remapped, 5 remapped+peering, 88 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.5 MiB/s wr, 495 op/s; 58/1644 objects degraded (3.528%); 2.3 MiB/s, 70 keys/s, 27 objects/s recovering 2024-09-18T07:37:21.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 261349 9.81 MB/sec execute 570 sec latency 46.461 ms 2024-09-18T07:37:21.896 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:21 smithi083 ceph-mon[63709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/1644 objects degraded (3.528%), 2 pgs degraded) 2024-09-18T07:37:21.897 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:21 smithi083 ceph-mon[63709]: Cluster is now healthy 2024-09-18T07:37:22.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:21 smithi096 ceph-mon[49483]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/1644 objects degraded (3.528%), 2 pgs degraded) 2024-09-18T07:37:22.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:21 smithi096 ceph-mon[49483]: Cluster is now healthy 2024-09-18T07:37:22.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 261678 9.80 MB/sec execute 571 sec latency 38.514 ms 2024-09-18T07:37:22.933 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:22 smithi083 ceph-mon[63709]: pgmap v287: 97 pgs: 2 peering, 95 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 6.4 MiB/s wr, 648 op/s; 2.8 MiB/s, 50 keys/s, 49 objects/s recovering 2024-09-18T07:37:23.092 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:22 smithi096 ceph-mon[49483]: pgmap v287: 97 pgs: 2 peering, 95 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 6.4 MiB/s wr, 648 op/s; 2.8 MiB/s, 50 keys/s, 49 objects/s recovering 2024-09-18T07:37:23.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 261974 9.81 MB/sec execute 572 sec latency 43.147 ms 2024-09-18T07:37:24.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 262330 9.81 MB/sec execute 573 sec latency 46.184 ms 2024-09-18T07:37:24.963 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:24 smithi083 ceph-mon[63709]: pgmap v288: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.0 MiB/s wr, 278 op/s; 2.2 MiB/s, 37 objects/s recovering 2024-09-18T07:37:25.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:24 smithi096 ceph-mon[49483]: pgmap v288: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.0 MiB/s wr, 278 op/s; 2.2 MiB/s, 37 objects/s recovering 2024-09-18T07:37:25.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 262785 9.81 MB/sec execute 574 sec latency 61.321 ms 2024-09-18T07:37:26.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 263059 9.81 MB/sec execute 575 sec latency 36.261 ms 2024-09-18T07:37:26.996 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:26 smithi083 ceph-mon[63709]: pgmap v289: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 2.3 MiB/s wr, 243 op/s; 2.0 MiB/s, 33 objects/s recovering 2024-09-18T07:37:26.996 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-18T07:37:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:26 smithi096 ceph-mon[49483]: pgmap v289: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 2.3 MiB/s wr, 243 op/s; 2.0 MiB/s, 33 objects/s recovering 2024-09-18T07:37:27.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-18T07:37:27.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 263334 9.81 MB/sec execute 576 sec latency 49.263 ms 2024-09-18T07:37:28.005 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:27 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-18T07:37:28.005 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:27 smithi083 ceph-mon[63709]: Upgrade: osd.7 is safe to restart 2024-09-18T07:37:28.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:27 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-18T07:37:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:27 smithi096 ceph-mon[49483]: Upgrade: osd.7 is safe to restart 2024-09-18T07:37:28.266 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:37:28.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 263694 9.81 MB/sec execute 577 sec latency 67.861 ms 2024-09-18T07:37:28.803 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:28 smithi083 ceph-mon[63709]: pgmap v290: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.4 MiB/s wr, 465 op/s; 1.6 MiB/s, 27 objects/s recovering 2024-09-18T07:37:28.803 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:28.804 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-18T07:37:28.804 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:37:28.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:28 smithi096 ceph-mon[49483]: pgmap v290: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.4 MiB/s wr, 465 op/s; 1.6 MiB/s, 27 objects/s recovering 2024-09-18T07:37:28.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:28.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-18T07:37:28.873 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:37:29.398 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:37:29.398 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (8m) 2m ago 18m 20.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:37:29.398 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (5m) 2m ago 18m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:37:29.398 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (5m) 22s ago 17m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:37:29.398 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (8m) 2m ago 17m 45.0M - dad864ee21e9 06ff04a18f50 2024-09-18T07:37:29.398 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (12m) 2m ago 12m 41.2M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:37:29.398 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (12m) 22s ago 12m 16.9M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:37:29.398 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (7m) 2m ago 19m 520M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (7m) 22s ago 17m 457M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (6m) 2m ago 19m 51.7M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (6m) 22s ago 17m 45.0M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (11m) 2m ago 11m 279M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (11m) 22s ago 11m 90.7M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (8m) 2m ago 17m 9.79M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (8m) 22s ago 17m 11.2M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 2m ago 16m 365M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4m) 2m ago 16m 277M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3m) 2m ago 15m 212M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (2m) 2m ago 15m 11.7M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (105s) 22s ago 14m 315M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2ca3dcb5baa0 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (66s) 22s ago 14m 278M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 8bd5b8b7b654 2024-09-18T07:37:29.399 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (27s) 22s ago 14m 12.0M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c3e35aae405e 2024-09-18T07:37:29.400 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (13m) 22s ago 13m 590M 2704M 17.2.0 e1d6a67b021e a437d69e9981 2024-09-18T07:37:29.400 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (7m) 2m ago 17m 53.3M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:37:29.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 264002 9.80 MB/sec execute 578 sec latency 68.148 ms 2024-09-18T07:37:29.828 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:29 smithi083 ceph-mon[63709]: from='client.44351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:37:29.829 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:29 smithi083 ceph-mon[63709]: Upgrade: Updating osd.7 2024-09-18T07:37:29.829 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:29 smithi083 ceph-mon[63709]: Deploying daemon osd.7 on smithi096 2024-09-18T07:37:29.829 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:29 smithi083 ceph-mon[63709]: from='client.44355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:37:30.045 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:37:30.045 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 7 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:37:30.046 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:37:30.047 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-18T07:37:30.047 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 11 2024-09-18T07:37:30.047 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:37:30.047 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:37:30.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:29 smithi096 ceph-mon[49483]: from='client.44351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:37:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:29 smithi096 ceph-mon[49483]: Upgrade: Updating osd.7 2024-09-18T07:37:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:29 smithi096 ceph-mon[49483]: Deploying daemon osd.7 on smithi096 2024-09-18T07:37:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:29 smithi096 ceph-mon[49483]: from='client.44355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:37:30.595 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:37:30.595 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:37:30.595 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "13/23 daemons upgraded", 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading osd daemons", 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:37:30.596 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:37:30.618 INFO:teuthology.orchestra.run.smithi083.stdout: 5 264244 9.80 MB/sec execute 579 sec latency 47.413 ms 2024-09-18T07:37:30.846 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:30 smithi083 ceph-mon[63709]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:37:30.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:30 smithi083 ceph-mon[63709]: pgmap v291: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 420 op/s; 1.4 MiB/s, 19 objects/s recovering 2024-09-18T07:37:30.847 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:30 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1437438527' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:31.009 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:30 smithi096 ceph-mon[49483]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:37:31.010 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:30 smithi096 ceph-mon[49483]: pgmap v291: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 420 op/s; 1.4 MiB/s, 19 objects/s recovering 2024-09-18T07:37:31.010 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:30 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1437438527' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:31.236 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:37:31.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 264475 9.79 MB/sec execute 580 sec latency 53.097 ms 2024-09-18T07:37:32.112 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:31 smithi083 ceph-mon[63709]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:37:32.113 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:32.113 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:37:32.113 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:32.113 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:31 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/829111617' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:37:32.324 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:31 smithi096 ceph-mon[49483]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:37:32.324 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:32.324 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:37:32.325 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:32.325 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:31 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/829111617' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:37:32.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 264896 9.79 MB/sec execute 581 sec latency 51.206 ms 2024-09-18T07:37:33.079 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:32 smithi096 systemd[1]: Stopping Ceph osd.7 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:37:33.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:33 smithi083 ceph-mon[63709]: pgmap v292: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.3 MiB/s wr, 533 op/s; 553 KiB/s, 14 objects/s recovering 2024-09-18T07:37:33.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:33 smithi096 ceph-mon[49483]: pgmap v292: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.3 MiB/s wr, 533 op/s; 553 KiB/s, 14 objects/s recovering 2024-09-18T07:37:33.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:33 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[39026]: 2024-09-18T07:37:33.209+0000 7fef74788700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:37:33.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:33 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[39026]: 2024-09-18T07:37:33.209+0000 7fef74788700 -1 osd.7 159 *** Got signal Terminated *** 2024-09-18T07:37:33.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:33 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[39026]: 2024-09-18T07:37:33.209+0000 7fef74788700 -1 osd.7 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:37:33.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 265264 9.79 MB/sec execute 582 sec latency 60.957 ms 2024-09-18T07:37:34.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:34 smithi083 ceph-mon[63709]: osd.7 marked itself down and dead 2024-09-18T07:37:34.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:34 smithi096 ceph-mon[49483]: osd.7 marked itself down and dead 2024-09-18T07:37:34.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 265404 9.78 MB/sec execute 583 sec latency 562.284 ms 2024-09-18T07:37:35.081 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:34 smithi096 podman[59033]: 2024-09-18 07:37:34.767879836 +0000 UTC m=+1.683809214 container died a437d69e99815e20d3c987d533208e8eda45ba19c9f8af2ab73cc4d897becbf9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, distribution-scope=public, architecture=x86_64, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux ) 2024-09-18T07:37:35.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:35 smithi083 ceph-mon[63709]: pgmap v293: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.2 MiB/s wr, 394 op/s 2024-09-18T07:37:35.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:35 smithi083 ceph-mon[63709]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:37:35.335 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:35 smithi083 ceph-mon[63709]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-18T07:37:35.336 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:35 smithi083 ceph-mon[63709]: osdmap e160: 8 total, 7 up, 8 in 2024-09-18T07:37:35.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:35 smithi096 ceph-mon[49483]: pgmap v293: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.2 MiB/s wr, 394 op/s 2024-09-18T07:37:35.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:35 smithi096 ceph-mon[49483]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-18T07:37:35.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:35 smithi096 ceph-mon[49483]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-18T07:37:35.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:35 smithi096 ceph-mon[49483]: osdmap e160: 8 total, 7 up, 8 in 2024-09-18T07:37:35.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:35 smithi096 podman[59033]: 2024-09-18 07:37:35.226816691 +0000 UTC m=+2.142746078 container cleanup a437d69e99815e20d3c987d533208e8eda45ba19c9f8af2ab73cc4d897becbf9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7, ceph=True, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, name=centos-stream, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_CLEAN=True, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, version=8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc.) 2024-09-18T07:37:35.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:35 smithi096 bash[59033]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7 2024-09-18T07:37:35.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 265610 9.78 MB/sec execute 584 sec latency 1040.269 ms 2024-09-18T07:37:35.871 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:35 smithi096 podman[59046]: 2024-09-18 07:37:35.545282107 +0000 UTC m=+0.772726188 container remove a437d69e99815e20d3c987d533208e8eda45ba19c9f8af2ab73cc4d897becbf9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, architecture=x86_64, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-18T07:37:36.093 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:36 smithi083 ceph-mon[63709]: osdmap e161: 8 total, 7 up, 8 in 2024-09-18T07:37:36.245 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:36 smithi096 ceph-mon[49483]: osdmap e161: 8 total, 7 up, 8 in 2024-09-18T07:37:36.245 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:35 smithi096 podman[59131]: 2024-09-18 07:37:35.850637577 +0000 UTC m=+0.019196843 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:37:36.245 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:35 smithi096 podman[59131]: 2024-09-18 07:37:35.980077024 +0000 UTC m=+0.148636289 container create 04d6676756515d932002c62a1cff9c2026cdba873c7bd784c2ac107e3baf0e9f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_CLEAN=True) 2024-09-18T07:37:36.246 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:36 smithi096 podman[59131]: 2024-09-18 07:37:36.242062803 +0000 UTC m=+0.410622087 container init 04d6676756515d932002c62a1cff9c2026cdba873c7bd784c2ac107e3baf0e9f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-18T07:37:36.622 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:36 smithi096 podman[59131]: 2024-09-18 07:37:36.246244614 +0000 UTC m=+0.414803890 container start 04d6676756515d932002c62a1cff9c2026cdba873c7bd784c2ac107e3baf0e9f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-18T07:37:36.622 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:36 smithi096 podman[59131]: 2024-09-18 07:37:36.342258691 +0000 UTC m=+0.510817968 container attach 04d6676756515d932002c62a1cff9c2026cdba873c7bd784c2ac107e3baf0e9f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:37:36.622 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:36 smithi096 podman[59131]: 2024-09-18 07:37:36.483361744 +0000 UTC m=+0.651921018 container died 04d6676756515d932002c62a1cff9c2026cdba873c7bd784c2ac107e3baf0e9f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-18T07:37:36.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 266191 9.78 MB/sec execute 585 sec latency 54.360 ms 2024-09-18T07:37:37.109 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:37 smithi083 ceph-mon[63709]: pgmap v296: 97 pgs: 1 peering, 13 stale+active+clean, 83 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.4 MiB/s wr, 307 op/s 2024-09-18T07:37:37.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:37 smithi096 ceph-mon[49483]: pgmap v296: 97 pgs: 1 peering, 13 stale+active+clean, 83 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.4 MiB/s wr, 307 op/s 2024-09-18T07:37:37.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:37 smithi096 podman[59152]: 2024-09-18 07:37:37.105687832 +0000 UTC m=+0.615604503 container remove 04d6676756515d932002c62a1cff9c2026cdba873c7bd784c2ac107e3baf0e9f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5) 2024-09-18T07:37:37.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:37 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-18T07:37:37.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:37 smithi096 systemd[1]: Stopped Ceph osd.7 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:37:37.373 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:37 smithi096 systemd[1]: ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.7.service: Consumed 6min 48.270s CPU time. 2024-09-18T07:37:37.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 266603 9.78 MB/sec execute 586 sec latency 35.525 ms 2024-09-18T07:37:37.658 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:37 smithi096 systemd[1]: Starting Ceph osd.7 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:37:37.659 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:37 smithi096 podman[59262]: 2024-09-18 07:37:37.517869531 +0000 UTC m=+0.019627729 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:37:37.912 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:37 smithi096 podman[59262]: 2024-09-18 07:37:37.656892253 +0000 UTC m=+0.158650442 container create 24991e6b81b7f33d2321d3413a2d35d767297027dc2685f5f7ddbe7b6b96bbfd (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-18T07:37:38.234 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:38 smithi083 ceph-mon[63709]: pgmap v297: 97 pgs: 5 active+undersized, 1 peering, 3 stale+active+clean, 23 active+undersized+degraded, 65 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 495 op/s; 178/1605 objects degraded (11.090%) 2024-09-18T07:37:38.234 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:38 smithi083 ceph-mon[63709]: Health check failed: Degraded data redundancy: 178/1605 objects degraded (11.090%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:38.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:38 smithi096 ceph-mon[49483]: pgmap v297: 97 pgs: 5 active+undersized, 1 peering, 3 stale+active+clean, 23 active+undersized+degraded, 65 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 495 op/s; 178/1605 objects degraded (11.090%) 2024-09-18T07:37:38.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:38 smithi096 ceph-mon[49483]: Health check failed: Degraded data redundancy: 178/1605 objects degraded (11.090%), 23 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:38.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:37 smithi096 podman[59262]: 2024-09-18 07:37:37.910407513 +0000 UTC m=+0.412165701 container init 24991e6b81b7f33d2321d3413a2d35d767297027dc2685f5f7ddbe7b6b96bbfd (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-18T07:37:38.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:37 smithi096 podman[59262]: 2024-09-18 07:37:37.915493063 +0000 UTC m=+0.417251247 container start 24991e6b81b7f33d2321d3413a2d35d767297027dc2685f5f7ddbe7b6b96bbfd (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-18T07:37:38.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:38 smithi096 podman[59262]: 2024-09-18 07:37:38.01016639 +0000 UTC m=+0.511924578 container attach 24991e6b81b7f33d2321d3413a2d35d767297027dc2685f5f7ddbe7b6b96bbfd (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731) 2024-09-18T07:37:38.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:38 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate[59277]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:37:38.373 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:38 smithi096 bash[59262]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:37:38.373 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:38 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate[59277]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:37:38.373 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:38 smithi096 bash[59262]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-18T07:37:38.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 267073 9.80 MB/sec execute 587 sec latency 50.885 ms 2024-09-18T07:37:39.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 267709 9.80 MB/sec execute 588 sec latency 40.319 ms 2024-09-18T07:37:40.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 268115 9.81 MB/sec execute 589 sec latency 48.053 ms 2024-09-18T07:37:40.745 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:40 smithi096 ceph-mon[49483]: pgmap v298: 97 pgs: 7 active+undersized, 1 peering, 34 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.6 MiB/s wr, 453 op/s; 241/1614 objects degraded (14.932%) 2024-09-18T07:37:40.745 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate[59277]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-18T07:37:40.745 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 bash[59262]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-18T07:37:40.745 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate[59277]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-18T07:37:40.745 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 bash[59262]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-18T07:37:40.745 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate[59277]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-18T07:37:40.746 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 bash[59262]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-18T07:37:40.899 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:40 smithi083 ceph-mon[63709]: pgmap v298: 97 pgs: 7 active+undersized, 1 peering, 34 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.6 MiB/s wr, 453 op/s; 241/1614 objects degraded (14.932%) 2024-09-18T07:37:41.090 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate[59277]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-18T07:37:41.090 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 bash[59262]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-18T07:37:41.091 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate[59277]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-18T07:37:41.091 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 bash[59262]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-18T07:37:41.091 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate[59277]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-18T07:37:41.091 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 bash[59262]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-18T07:37:41.091 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate[59277]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-18T07:37:41.091 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 bash[59262]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-18T07:37:41.091 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:40 smithi096 podman[59262]: 2024-09-18 07:37:40.776299713 +0000 UTC m=+3.278057902 container died 24991e6b81b7f33d2321d3413a2d35d767297027dc2685f5f7ddbe7b6b96bbfd (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-18T07:37:41.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 268535 9.80 MB/sec execute 590 sec latency 65.048 ms 2024-09-18T07:37:41.775 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:41 smithi096 podman[59448]: 2024-09-18 07:37:41.414748335 +0000 UTC m=+0.627103271 container remove 24991e6b81b7f33d2321d3413a2d35d767297027dc2685f5f7ddbe7b6b96bbfd (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-18T07:37:41.775 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:41 smithi096 podman[59506]: 2024-09-18 07:37:41.530906337 +0000 UTC m=+0.016606565 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:37:41.776 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:41 smithi096 podman[59506]: 2024-09-18 07:37:41.649392521 +0000 UTC m=+0.135092726 container create 18c8ec41919114925a61aea73a7860cd942f40363e6c714f1db599ed13dbf99a (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-18T07:37:42.052 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:41 smithi096 podman[59506]: 2024-09-18 07:37:41.885697354 +0000 UTC m=+0.371397574 container init 18c8ec41919114925a61aea73a7860cd942f40363e6c714f1db599ed13dbf99a (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:37:42.053 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:41 smithi096 podman[59506]: 2024-09-18 07:37:41.889012266 +0000 UTC m=+0.374712469 container start 18c8ec41919114925a61aea73a7860cd942f40363e6c714f1db599ed13dbf99a (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-18T07:37:42.053 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:41 smithi096 ceph-osd[59525]: -- 172.21.15.96:0/3105257273 <== mon.1 v2:172.21.15.96:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5565ca1661e0 con 0x5565c9305c00 2024-09-18T07:37:42.053 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:41 smithi096 bash[59506]: 18c8ec41919114925a61aea73a7860cd942f40363e6c714f1db599ed13dbf99a 2024-09-18T07:37:42.053 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:41 smithi096 systemd[1]: Started Ceph osd.7 for 126f471a-758e-11ef-bceb-c7b262605968. 2024-09-18T07:37:42.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 268857 9.80 MB/sec execute 591 sec latency 52.359 ms 2024-09-18T07:37:43.193 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:43 smithi083 ceph-mon[63709]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 MiB/s wr, 726 op/s; 263/1683 objects degraded (15.627%) 2024-09-18T07:37:43.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:43.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:43.194 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:37:43.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:43 smithi096 ceph-mon[49483]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 MiB/s wr, 726 op/s; 263/1683 objects degraded (15.627%) 2024-09-18T07:37:43.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:43.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:43.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:37:43.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 269250 9.81 MB/sec execute 592 sec latency 42.948 ms 2024-09-18T07:37:44.056 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:43 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:37:43.768+0000 7f9ea77ba640 -1 Falling back to public interface 2024-09-18T07:37:44.176 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:44 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 263/1656 objects degraded (15.882%), 35 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:44.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:44 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 263/1656 objects degraded (15.882%), 35 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:44.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 269623 9.81 MB/sec execute 593 sec latency 49.383 ms 2024-09-18T07:37:45.200 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:45 smithi083 ceph-mon[63709]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 601 op/s; 263/1656 objects degraded (15.882%) 2024-09-18T07:37:45.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:45 smithi096 ceph-mon[49483]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 601 op/s; 263/1656 objects degraded (15.882%) 2024-09-18T07:37:45.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 270007 9.81 MB/sec execute 594 sec latency 41.561 ms 2024-09-18T07:37:46.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 270193 9.80 MB/sec execute 595 sec latency 39.812 ms 2024-09-18T07:37:47.009 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:46 smithi083 ceph-mon[63709]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 529 op/s; 256/1650 objects degraded (15.515%) 2024-09-18T07:37:47.009 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:37:47.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.010 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.215 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:46 smithi096 ceph-mon[49483]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 529 op/s; 256/1650 objects degraded (15.515%) 2024-09-18T07:37:47.215 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.215 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:37:47.215 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.215 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.216 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.216 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:47.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 270477 9.80 MB/sec execute 596 sec latency 70.628 ms 2024-09-18T07:37:48.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 270822 9.80 MB/sec execute 597 sec latency 93.441 ms 2024-09-18T07:37:49.067 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:48 smithi083 ceph-mon[63709]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.6 MiB/s wr, 661 op/s; 260/1653 objects degraded (15.729%) 2024-09-18T07:37:49.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:48 smithi096 ceph-mon[49483]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.6 MiB/s wr, 661 op/s; 260/1653 objects degraded (15.729%) 2024-09-18T07:37:49.626 INFO:teuthology.orchestra.run.smithi083.stdout: 5 271150 9.80 MB/sec execute 598 sec latency 58.000 ms 2024-09-18T07:37:50.110 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:50 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 260/1662 objects degraded (15.644%), 35 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:50.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:50 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 260/1662 objects degraded (15.644%), 35 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:50.625 INFO:teuthology.orchestra.run.smithi083.stdout: 5 271416 9.80 MB/sec execute 599 sec latency 41.858 ms 2024-09-18T07:37:51.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.4 MiB/s wr, 537 op/s; 260/1662 objects degraded (15.644%) 2024-09-18T07:37:51.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:37:51.149 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-18T07:37:51.150 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-18T07:37:51.151 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:51 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-18T07:37:51.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.4 MiB/s wr, 537 op/s; 260/1662 objects degraded (15.644%) 2024-09-18T07:37:51.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-18T07:37:51.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-18T07:37:51.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:51 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-18T07:37:51.626 INFO:teuthology.orchestra.run.smithi083.stdout: 5 cleanup 600 sec 2024-09-18T07:37:51.885 INFO:teuthology.orchestra.run.smithi083.stdout: 0 cleanup 600 sec 2024-09-18T07:37:51.885 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:37:51.886 INFO:teuthology.orchestra.run.smithi083.stdout: Operation Count AvgLat MaxLat 2024-09-18T07:37:51.886 INFO:teuthology.orchestra.run.smithi083.stdout: ---------------------------------------- 2024-09-18T07:37:51.886 INFO:teuthology.orchestra.run.smithi083.stdout: NTCreateX 188769 4.536 2371.654 2024-09-18T07:37:51.886 INFO:teuthology.orchestra.run.smithi083.stdout: Close 138574 0.132 25.733 2024-09-18T07:37:51.886 INFO:teuthology.orchestra.run.smithi083.stdout: Rename 7994 8.411 1040.263 2024-09-18T07:37:51.886 INFO:teuthology.orchestra.run.smithi083.stdout: Unlink 38164 6.911 2373.630 2024-09-18T07:37:51.886 INFO:teuthology.orchestra.run.smithi083.stdout: Qpathinfo 171129 1.156 92.597 2024-09-18T07:37:51.886 INFO:teuthology.orchestra.run.smithi083.stdout: Qfileinfo 29811 0.006 0.962 2024-09-18T07:37:51.886 INFO:teuthology.orchestra.run.smithi083.stdout: Qfsinfo 31347 0.612 18.571 2024-09-18T07:37:51.887 INFO:teuthology.orchestra.run.smithi083.stdout: Sfileinfo 15400 12.918 1066.741 2024-09-18T07:37:51.887 INFO:teuthology.orchestra.run.smithi083.stdout: Find 66066 0.929 39.812 2024-09-18T07:37:51.887 INFO:teuthology.orchestra.run.smithi083.stdout: WriteX 93156 14.069 2377.482 2024-09-18T07:37:51.887 INFO:teuthology.orchestra.run.smithi083.stdout: ReadX 295666 0.010 6.829 2024-09-18T07:37:51.887 INFO:teuthology.orchestra.run.smithi083.stdout: LockX 612 0.377 5.838 2024-09-18T07:37:51.887 INFO:teuthology.orchestra.run.smithi083.stdout: UnlockX 612 0.315 2.135 2024-09-18T07:37:51.887 INFO:teuthology.orchestra.run.smithi083.stdout: Flush 13220 0.002 0.228 2024-09-18T07:37:51.887 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:37:51.887 INFO:teuthology.orchestra.run.smithi083.stdout:Throughput 9.79502 MB/sec 5 clients 5 procs max_latency=2377.486 ms 2024-09-18T07:37:51.889 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-18T07:37:51.923 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-18T07:37:52.049 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-18T07:37:52.076 INFO:teuthology.orchestra.run.smithi083.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-18T07:37:52.076 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:37:52.076 INFO:teuthology.orchestra.run.smithi083.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-18T07:37:52.076 INFO:teuthology.orchestra.run.smithi083.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-18T07:37:52.177 INFO:teuthology.orchestra.run.smithi083.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-18T07:37:52.177 INFO:teuthology.orchestra.run.smithi083.stdout:releasing clients 2024-09-18T07:37:52.177 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:52 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all osd 2024-09-18T07:37:52.177 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:52 smithi083 ceph-mon[63709]: Upgrade: Setting require_osd_release to 19 squid 2024-09-18T07:37:52.177 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:52 smithi083 ceph-mon[63709]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-18T07:37:52.178 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-18T07:37:52.178 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:52 smithi083 ceph-mon[63709]: osdmap e162: 8 total, 7 up, 8 in 2024-09-18T07:37:52.178 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi083.kjbbry"]} : dispatch 2024-09-18T07:37:52.322 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:52 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all osd 2024-09-18T07:37:52.322 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:52 smithi096 ceph-mon[49483]: Upgrade: Setting require_osd_release to 19 squid 2024-09-18T07:37:52.322 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:52 smithi096 ceph-mon[49483]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-18T07:37:52.322 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-18T07:37:52.322 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:52 smithi096 ceph-mon[49483]: osdmap e162: 8 total, 7 up, 8 in 2024-09-18T07:37:52.322 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi083.kjbbry"]} : dispatch 2024-09-18T07:37:53.207 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:53 smithi083 ceph-mon[63709]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 596 op/s; 245/1629 objects degraded (15.040%) 2024-09-18T07:37:53.208 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:53 smithi083 ceph-mon[63709]: Upgrade: It appears safe to stop mds.foofs.smithi083.kjbbry 2024-09-18T07:37:53.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:53 smithi096 ceph-mon[49483]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 596 op/s; 245/1629 objects degraded (15.040%) 2024-09-18T07:37:53.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:53 smithi096 ceph-mon[49483]: Upgrade: It appears safe to stop mds.foofs.smithi083.kjbbry 2024-09-18T07:37:54.188 INFO:teuthology.orchestra.run.smithi083.stdout: 5 390 26.75 MB/sec execute 1 sec latency 169.710 ms 2024-09-18T07:37:54.621 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:54 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:37:54.156+0000 7f9ea77ba640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-18T07:37:55.197 INFO:teuthology.orchestra.run.smithi083.stdout: 5 614 29.29 MB/sec execute 2 sec latency 29.497 ms 2024-09-18T07:37:55.270 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:55 smithi083 ceph-mon[63709]: pgmap v306: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.4 MiB/s wr, 506 op/s; 246/1647 objects degraded (14.936%) 2024-09-18T07:37:55.271 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:55 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:55.271 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:55 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.kjbbry", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-18T07:37:55.271 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:55 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:37:55.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:55 smithi096 ceph-mon[49483]: pgmap v306: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.4 MiB/s wr, 506 op/s; 246/1647 objects degraded (14.936%) 2024-09-18T07:37:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:55 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:37:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:55 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi083.kjbbry", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-18T07:37:55.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:55 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:37:56.038 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:56 smithi083 ceph-mon[63709]: Upgrade: Updating mds.foofs.smithi083.kjbbry 2024-09-18T07:37:56.197 INFO:teuthology.orchestra.run.smithi083.stdout: 5 935 22.01 MB/sec execute 3 sec latency 63.307 ms 2024-09-18T07:37:56.316 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:56 smithi083 ceph-mon[63709]: Deploying daemon mds.foofs.smithi083.kjbbry on smithi083 2024-09-18T07:37:56.317 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:56 smithi083 ceph-mon[63709]: from='osd.7 [v2:172.21.15.96:6824/412219063,v1:172.21.15.96:6825/412219063]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-18T07:37:56.317 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:56 smithi083 ceph-mon[63709]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-18T07:37:56.371 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:55 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:37:55.892+0000 7f9ea77ba640 -1 osd.7 159 log_to_monitors true 2024-09-18T07:37:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:56 smithi096 ceph-mon[49483]: Upgrade: Updating mds.foofs.smithi083.kjbbry 2024-09-18T07:37:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:56 smithi096 ceph-mon[49483]: Deploying daemon mds.foofs.smithi083.kjbbry on smithi083 2024-09-18T07:37:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:56 smithi096 ceph-mon[49483]: from='osd.7 [v2:172.21.15.96:6824/412219063,v1:172.21.15.96:6825/412219063]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-18T07:37:56.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:56 smithi096 ceph-mon[49483]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-18T07:37:56.871 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:37:56 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:37:56.511+0000 7f9e9b55c640 -1 osd.7 159 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T07:37:57.095 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:57 smithi083 ceph-mon[63709]: pgmap v307: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.3 MiB/s wr, 502 op/s; 231/1500 objects degraded (15.400%) 2024-09-18T07:37:57.095 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:57 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 231/1500 objects degraded (15.400%), 35 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:57.095 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:57 smithi083 ceph-mon[63709]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-18T07:37:57.095 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:57 smithi083 ceph-mon[63709]: from='osd.7 [v2:172.21.15.96:6824/412219063,v1:172.21.15.96:6825/412219063]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:37:57.095 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:57 smithi083 ceph-mon[63709]: osdmap e163: 8 total, 7 up, 8 in 2024-09-18T07:37:57.095 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:57 smithi083 ceph-mon[63709]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:37:57.096 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:57 smithi083 ceph-mon[63709]: from='osd.7 ' entity='osd.7' 2024-09-18T07:37:57.197 INFO:teuthology.orchestra.run.smithi083.stdout: 5 1264 18.88 MB/sec execute 4 sec latency 131.002 ms 2024-09-18T07:37:57.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:57 smithi096 ceph-mon[49483]: pgmap v307: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.3 MiB/s wr, 502 op/s; 231/1500 objects degraded (15.400%) 2024-09-18T07:37:57.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:57 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 231/1500 objects degraded (15.400%), 35 pgs degraded (PG_DEGRADED) 2024-09-18T07:37:57.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:57 smithi096 ceph-mon[49483]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-18T07:37:57.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:57 smithi096 ceph-mon[49483]: from='osd.7 [v2:172.21.15.96:6824/412219063,v1:172.21.15.96:6825/412219063]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:37:57.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:57 smithi096 ceph-mon[49483]: osdmap e163: 8 total, 7 up, 8 in 2024-09-18T07:37:57.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:57 smithi096 ceph-mon[49483]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi096", "root=default"]} : dispatch 2024-09-18T07:37:57.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:57 smithi096 ceph-mon[49483]: from='osd.7 ' entity='osd.7' 2024-09-18T07:37:57.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:57 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083[63705]: 2024-09-18T07:37:57.392+0000 7fcf66eee640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-18T07:37:58.197 INFO:teuthology.orchestra.run.smithi083.stdout: 5 cleanup 5 sec 2024-09-18T07:37:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:37:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-18T07:37:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-18T07:37:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: osd.7 [v2:172.21.15.96:6824/412219063,v1:172.21.15.96:6825/412219063] boot 2024-09-18T07:37:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: osdmap e164: 8 total, 8 up, 8 in 2024-09-18T07:37:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: Standby daemon mds.foofs.smithi096.bzilgv assigned to filesystem foofs as rank 0 2024-09-18T07:37:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-18T07:37:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-18T07:37:58.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-18T07:37:58.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-18T07:37:58.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: fsmap foofs:1/1 {0=foofs.smithi096.bzilgv=up:replay} 2024-09-18T07:37:58.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:58 smithi083 ceph-mon[63709]: osdmap e165: 8 total, 8 up, 8 in 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: osd.7 [v2:172.21.15.96:6824/412219063,v1:172.21.15.96:6825/412219063] boot 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: osdmap e164: 8 total, 8 up, 8 in 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: Standby daemon mds.foofs.smithi096.bzilgv assigned to filesystem foofs as rank 0 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: fsmap foofs:1/1 {0=foofs.smithi096.bzilgv=up:replay} 2024-09-18T07:37:58.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-mon[49483]: osdmap e165: 8 total, 8 up, 8 in 2024-09-18T07:37:59.041 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:37:58.677+0000 7fb7cc35f640 -1 bdev(0x5599e0becc00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-18T07:37:59.041 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:37:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:37:58.677+0000 7fb7cf365640 -1 bdev(0x5599e0becc00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-18T07:37:59.197 INFO:teuthology.orchestra.run.smithi083.stdout: 5 cleanup 6 sec 2024-09-18T07:37:59.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:59 smithi083 ceph-mon[63709]: pgmap v311: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 17 MiB/s wr, 318 op/s; 196/1305 objects degraded (15.019%) 2024-09-18T07:37:59.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:37:59 smithi083 ceph-mon[63709]: osdmap e166: 8 total, 8 up, 8 in 2024-09-18T07:37:59.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:59 smithi096 ceph-mon[49483]: pgmap v311: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 17 MiB/s wr, 318 op/s; 196/1305 objects degraded (15.019%) 2024-09-18T07:37:59.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:37:59 smithi096 ceph-mon[49483]: osdmap e166: 8 total, 8 up, 8 in 2024-09-18T07:38:00.198 INFO:teuthology.orchestra.run.smithi083.stdout: 5 cleanup 7 sec 2024-09-18T07:38:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:00 smithi083 ceph-mon[63709]: osdmap e167: 8 total, 8 up, 8 in 2024-09-18T07:38:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:00 smithi083 ceph-mon[63709]: pgmap v314: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+undersized, 26 active+undersized+degraded, 64 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 50 op/s; 215/1368 objects degraded (15.716%); 421 KiB/s, 11 objects/s recovering 2024-09-18T07:38:00.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:00 smithi083 ceph-mon[63709]: reconnect by client.24337 172.21.15.96:0/3655200669 after 0 2024-09-18T07:38:00.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:00 smithi083 ceph-mon[63709]: reconnect by client.14562 172.21.15.83:0/473556564 after 0.000999978 2024-09-18T07:38:00.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:00 smithi083 ceph-mon[63709]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:reconnect 2024-09-18T07:38:00.869 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:00 smithi083 ceph-mon[63709]: fsmap foofs:1/1 {0=foofs.smithi096.bzilgv=up:reconnect} 2024-09-18T07:38:00.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:00 smithi096 ceph-mon[49483]: osdmap e167: 8 total, 8 up, 8 in 2024-09-18T07:38:00.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:00 smithi096 ceph-mon[49483]: pgmap v314: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+undersized, 26 active+undersized+degraded, 64 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 50 op/s; 215/1368 objects degraded (15.716%); 421 KiB/s, 11 objects/s recovering 2024-09-18T07:38:00.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:00 smithi096 ceph-mon[49483]: reconnect by client.24337 172.21.15.96:0/3655200669 after 0 2024-09-18T07:38:00.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:00 smithi096 ceph-mon[49483]: reconnect by client.14562 172.21.15.83:0/473556564 after 0.000999978 2024-09-18T07:38:00.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:00 smithi096 ceph-mon[49483]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:reconnect 2024-09-18T07:38:00.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:00 smithi096 ceph-mon[49483]: fsmap foofs:1/1 {0=foofs.smithi096.bzilgv=up:reconnect} 2024-09-18T07:38:01.198 INFO:teuthology.orchestra.run.smithi083.stdout: 5 cleanup 8 sec 2024-09-18T07:38:01.631 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:38:02.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:01 smithi083 ceph-mon[63709]: osdmap e168: 8 total, 8 up, 8 in 2024-09-18T07:38:02.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:02.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:38:02.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:01 smithi083 ceph-mon[63709]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:rejoin 2024-09-18T07:38:02.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:01 smithi083 ceph-mon[63709]: fsmap foofs:1/1 {0=foofs.smithi096.bzilgv=up:rejoin} 2024-09-18T07:38:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:01 smithi096 ceph-mon[49483]: osdmap e168: 8 total, 8 up, 8 in 2024-09-18T07:38:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:38:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:01 smithi096 ceph-mon[49483]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:rejoin 2024-09-18T07:38:02.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:01 smithi096 ceph-mon[49483]: fsmap foofs:1/1 {0=foofs.smithi096.bzilgv=up:rejoin} 2024-09-18T07:38:02.198 INFO:teuthology.orchestra.run.smithi083.stdout: 5 cleanup 9 sec 2024-09-18T07:38:02.348 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:38:02.348 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (9m) 2m ago 18m 20.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (6m) 2m ago 18m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (6m) 16s ago 18m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (9m) 2m ago 18m 45.0M - dad864ee21e9 06ff04a18f50 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (12m) 2m ago 12m 41.2M - 17.2.0 e1d6a67b021e 0bc03d423c57 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (12m) 16s ago 12m 17.0M - 17.2.0 e1d6a67b021e 9090466929b3 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (8m) 2m ago 19m 520M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (7m) 16s ago 18m 458M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (7m) 2m ago 20m 51.7M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (6m) 16s ago 17m 46.9M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (12m) 2m ago 12m 279M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (12m) 16s ago 12m 90.7M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (9m) 2m ago 18m 9.79M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:38:02.349 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (8m) 16s ago 17m 11.9M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:38:02.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (5m) 2m ago 17m 365M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:38:02.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (4m) 2m ago 16m 277M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:38:02.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (3m) 2m ago 16m 212M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:38:02.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3m) 2m ago 15m 11.7M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:38:02.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (2m) 16s ago 15m 331M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2ca3dcb5baa0 2024-09-18T07:38:02.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (99s) 16s ago 14m 328M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 8bd5b8b7b654 2024-09-18T07:38:02.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (60s) 16s ago 14m 277M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c3e35aae405e 2024-09-18T07:38:02.350 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (20s) 16s ago 14m 12.1M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 18c8ec419191 2024-09-18T07:38:02.350 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (7m) 2m ago 17m 53.3M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:38:02.773 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:38:02.773 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:38:02.773 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:38:02.773 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:38:02.773 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:38:02.773 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:38:02.773 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:38:02.773 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 8 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 12 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:38:02.774 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:38:03.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:02 smithi083 ceph-mon[63709]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:03.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:02 smithi083 ceph-mon[63709]: osdmap e169: 8 total, 8 up, 8 in 2024-09-18T07:38:03.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:02 smithi083 ceph-mon[63709]: pgmap v317: 97 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s rd, 5.2 MiB/s wr, 436 op/s; 51/1443 objects degraded (3.534%); 11 MiB/s, 96 objects/s recovering 2024-09-18T07:38:03.052 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:02 smithi083 ceph-mon[63709]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:03.053 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:02 smithi083 ceph-mon[63709]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:clientreplay 2024-09-18T07:38:03.053 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:02 smithi083 ceph-mon[63709]: fsmap foofs:1/1 {0=foofs.smithi096.bzilgv=up:clientreplay} 2024-09-18T07:38:03.053 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:02 smithi083 ceph-mon[63709]: daemon mds.foofs.smithi096.bzilgv is now active in filesystem foofs as rank 0 2024-09-18T07:38:03.053 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:02 smithi083 ceph-mon[63709]: Health check update: Degraded data redundancy: 215/1368 objects degraded (15.716%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:38:03.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:02 smithi096 ceph-mon[49483]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:02 smithi096 ceph-mon[49483]: osdmap e169: 8 total, 8 up, 8 in 2024-09-18T07:38:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:02 smithi096 ceph-mon[49483]: pgmap v317: 97 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s rd, 5.2 MiB/s wr, 436 op/s; 51/1443 objects degraded (3.534%); 11 MiB/s, 96 objects/s recovering 2024-09-18T07:38:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:02 smithi096 ceph-mon[49483]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:02 smithi096 ceph-mon[49483]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:clientreplay 2024-09-18T07:38:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:02 smithi096 ceph-mon[49483]: fsmap foofs:1/1 {0=foofs.smithi096.bzilgv=up:clientreplay} 2024-09-18T07:38:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:02 smithi096 ceph-mon[49483]: daemon mds.foofs.smithi096.bzilgv is now active in filesystem foofs as rank 0 2024-09-18T07:38:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:02 smithi096 ceph-mon[49483]: Health check update: Degraded data redundancy: 215/1368 objects degraded (15.716%), 27 pgs degraded (PG_DEGRADED) 2024-09-18T07:38:03.168 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "osd", 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "14/23 daemons upgraded", 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mds daemons", 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:38:03.169 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:38:03.198 INFO:teuthology.orchestra.run.smithi083.stdout: 5 cleanup 10 sec 2024-09-18T07:38:03.325 INFO:teuthology.orchestra.run.smithi083.stdout: 0 cleanup 10 sec 2024-09-18T07:38:03.325 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:38:03.325 INFO:teuthology.orchestra.run.smithi083.stdout: Operation Count AvgLat MaxLat 2024-09-18T07:38:03.325 INFO:teuthology.orchestra.run.smithi083.stdout: ---------------------------------------- 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: NTCreateX 797 14.382 5673.320 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: Close 729 14.083 5681.436 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: Rename 50 14.643 130.996 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: Unlink 83 2.969 15.864 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: Qpathinfo 472 1.494 50.389 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: Qfileinfo 348 62.922 5671.737 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: Qfsinfo 159 0.512 2.987 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: Sfileinfo 166 13.038 72.425 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: Find 220 0.848 6.619 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: WriteX 1625 0.025 0.068 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: ReadX 938 0.019 1.941 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: Flush 120 12.576 65.382 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:38:03.326 INFO:teuthology.orchestra.run.smithi083.stdout:Throughput 18.8779 MB/sec 5 clients 5 procs max_latency=169.710 ms 2024-09-18T07:38:03.328 DEBUG:teuthology.parallel:result is None 2024-09-18T07:38:03.648 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_WARN insufficient standby MDS daemons available; Degraded data redundancy: 51/1443 objects degraded (3.534%), 1 pg degraded 2024-09-18T07:38:03.648 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-18T07:38:03.648 INFO:teuthology.orchestra.run.smithi083.stdout: have 0; want 1 more 2024-09-18T07:38:03.648 INFO:teuthology.orchestra.run.smithi083.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 51/1443 objects degraded (3.534%), 1 pg degraded 2024-09-18T07:38:03.649 INFO:teuthology.orchestra.run.smithi083.stdout: pg 3.16 is active+recovery_wait+undersized+degraded+remapped, acting [5,1] 2024-09-18T07:38:04.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:03 smithi083 ceph-mon[63709]: from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:04.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:03 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/2269456911' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:04.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:03 smithi083 ceph-mon[63709]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-18T07:38:04.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:03 smithi083 ceph-mon[63709]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:active 2024-09-18T07:38:04.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:03 smithi083 ceph-mon[63709]: fsmap foofs:1 {0=foofs.smithi096.bzilgv=up:active} 2024-09-18T07:38:04.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:03 smithi096 ceph-mon[49483]: from='client.44381 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:04.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:03 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/2269456911' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:04.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:03 smithi096 ceph-mon[49483]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-18T07:38:04.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:03 smithi096 ceph-mon[49483]: mds.? [v2:172.21.15.96:6832/3136855526,v1:172.21.15.96:6833/3136855526] up:active 2024-09-18T07:38:04.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:03 smithi096 ceph-mon[49483]: fsmap foofs:1 {0=foofs.smithi096.bzilgv=up:active} 2024-09-18T07:38:05.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:04 smithi083 ceph-mon[63709]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:05.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:04 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1455788930' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:38:05.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:04 smithi083 ceph-mon[63709]: pgmap v318: 97 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 3.3 MiB/s wr, 337 op/s; 51/1440 objects degraded (3.542%); 8.7 MiB/s, 76 objects/s recovering 2024-09-18T07:38:05.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:04 smithi096 ceph-mon[49483]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:05.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:04 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1455788930' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:38:05.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:04 smithi096 ceph-mon[49483]: pgmap v318: 97 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 3.3 MiB/s wr, 337 op/s; 51/1440 objects degraded (3.542%); 8.7 MiB/s, 76 objects/s recovering 2024-09-18T07:38:06.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:05 smithi083 ceph-mon[63709]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 51/1440 objects degraded (3.542%), 1 pg degraded) 2024-09-18T07:38:06.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:05 smithi096 ceph-mon[49483]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 51/1440 objects degraded (3.542%), 1 pg degraded) 2024-09-18T07:38:06.866 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:06 smithi083 ceph-mon[63709]: pgmap v319: 97 pgs: 4 peering, 93 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 1.7 MiB/s wr, 275 op/s; 7.8 MiB/s, 65 objects/s recovering 2024-09-18T07:38:07.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:06 smithi096 ceph-mon[49483]: pgmap v319: 97 pgs: 4 peering, 93 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 1.7 MiB/s wr, 275 op/s; 7.8 MiB/s, 65 objects/s recovering 2024-09-18T07:38:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:08 smithi083 ceph-mon[63709]: pgmap v320: 97 pgs: 97 active+clean; 282 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.5 MiB/s wr, 260 op/s; 6.8 MiB/s, 59 objects/s recovering 2024-09-18T07:38:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:08 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:09.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:08 smithi096 ceph-mon[49483]: pgmap v320: 97 pgs: 97 active+clean; 282 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.5 MiB/s wr, 260 op/s; 6.8 MiB/s, 59 objects/s recovering 2024-09-18T07:38:09.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:09.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:09.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:08 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:10.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:09 smithi083 ceph-mon[63709]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-18T07:38:10.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:09 smithi083 ceph-mon[63709]: Cluster is now healthy 2024-09-18T07:38:10.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:09 smithi083 ceph-mon[63709]: mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617] up:boot 2024-09-18T07:38:10.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:09 smithi083 ceph-mon[63709]: fsmap foofs:1 {0=foofs.smithi096.bzilgv=up:active} 1 up:standby 2024-09-18T07:38:10.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:09 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"} : dispatch 2024-09-18T07:38:10.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:09 smithi096 ceph-mon[49483]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-18T07:38:10.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:09 smithi096 ceph-mon[49483]: Cluster is now healthy 2024-09-18T07:38:10.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:09 smithi096 ceph-mon[49483]: mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617] up:boot 2024-09-18T07:38:10.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:09 smithi096 ceph-mon[49483]: fsmap foofs:1 {0=foofs.smithi096.bzilgv=up:active} 1 up:standby 2024-09-18T07:38:10.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:09 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata", "who": "foofs.smithi083.kjbbry"} : dispatch 2024-09-18T07:38:10.867 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:10 smithi083 ceph-mon[63709]: pgmap v321: 97 pgs: 97 active+clean; 262 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.3 MiB/s wr, 239 op/s; 6.0 MiB/s, 52 objects/s recovering 2024-09-18T07:38:11.045 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:10 smithi096 ceph-mon[49483]: pgmap v321: 97 pgs: 97 active+clean; 262 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.3 MiB/s wr, 239 op/s; 6.0 MiB/s, 52 objects/s recovering 2024-09-18T07:38:13.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:12 smithi083 ceph-mon[63709]: pgmap v322: 97 pgs: 97 active+clean; 242 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 248 KiB/s wr, 71 op/s; 1.2 MiB/s, 12 objects/s recovering 2024-09-18T07:38:13.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:12 smithi096 ceph-mon[49483]: pgmap v322: 97 pgs: 97 active+clean; 242 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 248 KiB/s wr, 71 op/s; 1.2 MiB/s, 12 objects/s recovering 2024-09-18T07:38:15.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:14 smithi083 ceph-mon[63709]: pgmap v323: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 208 KiB/s wr, 60 op/s; 995 KiB/s, 10 objects/s recovering 2024-09-18T07:38:15.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:15.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:14 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:15.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:14 smithi096 ceph-mon[49483]: pgmap v323: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 208 KiB/s wr, 60 op/s; 995 KiB/s, 10 objects/s recovering 2024-09-18T07:38:15.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:15.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:14 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:16.073 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:16.074 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:16.074 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:16.074 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:38:16.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:16.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:16.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:16.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:38:17.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:17 smithi083 ceph-mon[63709]: pgmap v324: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 208 KiB/s wr, 57 op/s; 995 KiB/s, 10 objects/s recovering 2024-09-18T07:38:17.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:17 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:17.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:17 smithi096 ceph-mon[49483]: pgmap v324: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 208 KiB/s wr, 57 op/s; 995 KiB/s, 10 objects/s recovering 2024-09-18T07:38:17.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:17 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:19.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: pgmap v325: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 210 KiB/s wr, 52 op/s; 590 KiB/s, 6 objects/s recovering 2024-09-18T07:38:19.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:19.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:19.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:19.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:38:19.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:19.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:19.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:19.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:19.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:19.370 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:19.370 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:19.370 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:19 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi096.bzilgv"]} : dispatch 2024-09-18T07:38:19.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: pgmap v325: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 210 KiB/s wr, 52 op/s; 590 KiB/s, 6 objects/s recovering 2024-09-18T07:38:19.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:19.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:19.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:19.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:38:19.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:19.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:19.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:19.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:19.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:19.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:19.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:19.374 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:19 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi096.bzilgv"]} : dispatch 2024-09-18T07:38:20.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:20 smithi083 ceph-mon[63709]: Detected new or changed devices on smithi083 2024-09-18T07:38:20.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:20 smithi083 ceph-mon[63709]: Upgrade: It appears safe to stop mds.foofs.smithi096.bzilgv 2024-09-18T07:38:20.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:20 smithi096 ceph-mon[49483]: Detected new or changed devices on smithi083 2024-09-18T07:38:20.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:20 smithi096 ceph-mon[49483]: Upgrade: It appears safe to stop mds.foofs.smithi096.bzilgv 2024-09-18T07:38:21.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:21 smithi083 ceph-mon[63709]: pgmap v326: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 59 KiB/s wr, 17 op/s 2024-09-18T07:38:21.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:21.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi096.bzilgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-18T07:38:21.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:21 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:21.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:21 smithi096 ceph-mon[49483]: pgmap v326: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 59 KiB/s wr, 17 op/s 2024-09-18T07:38:21.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:21.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi096.bzilgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-18T07:38:21.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:21 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:22.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083[63705]: 2024-09-18T07:38:22.028+0000 7fcf66eee640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-18T07:38:22.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: Upgrade: Updating mds.foofs.smithi096.bzilgv 2024-09-18T07:38:22.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: Deploying daemon mds.foofs.smithi096.bzilgv on smithi096 2024-09-18T07:38:22.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-18T07:38:22.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-18T07:38:22.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: osdmap e170: 8 total, 8 up, 8 in 2024-09-18T07:38:22.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: Standby daemon mds.foofs.smithi083.kjbbry assigned to filesystem foofs as rank 0 2024-09-18T07:38:22.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-18T07:38:22.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-18T07:38:22.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-18T07:38:22.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:22 smithi083 ceph-mon[63709]: fsmap foofs:1/1 {0=foofs.smithi083.kjbbry=up:replay} 2024-09-18T07:38:22.371 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:38:22.054+0000 7fb7c8357640 -1 bdev(0x5599e0becc00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-18T07:38:22.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:38:22.049+0000 7f9e83512640 -1 bdev(0x5565ca15ac00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 4 2024-09-18T07:38:22.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:38:22.050+0000 7f9e8751a640 -1 bdev(0x5565ca15ac00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 4 2024-09-18T07:38:22.372 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:38:22.051+0000 7f9e89d1f640 -1 bdev(0x5565ca15ac00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 5 2024-09-18T07:38:22.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: Upgrade: Updating mds.foofs.smithi096.bzilgv 2024-09-18T07:38:22.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: Deploying daemon mds.foofs.smithi096.bzilgv on smithi096 2024-09-18T07:38:22.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-18T07:38:22.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-18T07:38:22.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: osdmap e170: 8 total, 8 up, 8 in 2024-09-18T07:38:22.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: Standby daemon mds.foofs.smithi083.kjbbry assigned to filesystem foofs as rank 0 2024-09-18T07:38:22.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-18T07:38:22.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-18T07:38:22.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-18T07:38:22.373 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:22 smithi096 ceph-mon[49483]: fsmap foofs:1/1 {0=foofs.smithi083.kjbbry=up:replay} 2024-09-18T07:38:23.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:23 smithi083 ceph-mon[63709]: pgmap v327: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 56 KiB/s wr, 10 op/s 2024-09-18T07:38:23.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:23 smithi096 ceph-mon[49483]: pgmap v327: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 56 KiB/s wr, 10 op/s 2024-09-18T07:38:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:24 smithi096 ceph-mon[49483]: reconnect by client.14562 172.21.15.83:0/473556564 after 0 2024-09-18T07:38:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:24 smithi096 ceph-mon[49483]: reconnect by client.24337 172.21.15.96:0/3655200669 after 0 2024-09-18T07:38:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:24 smithi096 ceph-mon[49483]: mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617] up:reconnect 2024-09-18T07:38:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:24 smithi096 ceph-mon[49483]: fsmap foofs:1/1 {0=foofs.smithi083.kjbbry=up:reconnect} 2024-09-18T07:38:24.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:24 smithi096 ceph-mon[49483]: pgmap v329: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.1 KiB/s wr, 1 op/s 2024-09-18T07:38:24.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:24 smithi083 ceph-mon[63709]: reconnect by client.14562 172.21.15.83:0/473556564 after 0 2024-09-18T07:38:24.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:24 smithi083 ceph-mon[63709]: reconnect by client.24337 172.21.15.96:0/3655200669 after 0 2024-09-18T07:38:24.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:24 smithi083 ceph-mon[63709]: mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617] up:reconnect 2024-09-18T07:38:24.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:24 smithi083 ceph-mon[63709]: fsmap foofs:1/1 {0=foofs.smithi083.kjbbry=up:reconnect} 2024-09-18T07:38:24.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:24 smithi083 ceph-mon[63709]: pgmap v329: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.1 KiB/s wr, 1 op/s 2024-09-18T07:38:25.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:25 smithi096 ceph-mon[49483]: mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617] up:rejoin 2024-09-18T07:38:25.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:25 smithi096 ceph-mon[49483]: fsmap foofs:1/1 {0=foofs.smithi083.kjbbry=up:rejoin} 2024-09-18T07:38:25.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:25 smithi096 ceph-mon[49483]: daemon mds.foofs.smithi083.kjbbry is now active in filesystem foofs as rank 0 2024-09-18T07:38:25.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:25 smithi083 ceph-mon[63709]: mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617] up:rejoin 2024-09-18T07:38:25.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:25 smithi083 ceph-mon[63709]: fsmap foofs:1/1 {0=foofs.smithi083.kjbbry=up:rejoin} 2024-09-18T07:38:25.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:25 smithi083 ceph-mon[63709]: daemon mds.foofs.smithi083.kjbbry is now active in filesystem foofs as rank 0 2024-09-18T07:38:26.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:26 smithi083 ceph-mon[63709]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-18T07:38:26.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:26 smithi083 ceph-mon[63709]: mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617] up:active 2024-09-18T07:38:26.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:26 smithi083 ceph-mon[63709]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 2024-09-18T07:38:26.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:26 smithi083 ceph-mon[63709]: pgmap v330: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 3.1 KiB/s wr, 3 op/s 2024-09-18T07:38:26.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:26 smithi096 ceph-mon[49483]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-18T07:38:26.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:26 smithi096 ceph-mon[49483]: mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617] up:active 2024-09-18T07:38:26.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:26 smithi096 ceph-mon[49483]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 2024-09-18T07:38:26.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:26 smithi096 ceph-mon[49483]: pgmap v330: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 3.1 KiB/s wr, 3 op/s 2024-09-18T07:38:28.955 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:28 smithi096 ceph-mon[49483]: pgmap v331: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 102 B/s wr, 8 op/s 2024-09-18T07:38:29.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:28 smithi083 ceph-mon[63709]: pgmap v331: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 102 B/s wr, 8 op/s 2024-09-18T07:38:30.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:29 smithi083 ceph-mon[63709]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-18T07:38:30.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:29 smithi083 ceph-mon[63709]: Cluster is now healthy 2024-09-18T07:38:30.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:29 smithi083 ceph-mon[63709]: mds.? [v2:172.21.15.96:6832/824261772,v1:172.21.15.96:6833/824261772] up:boot 2024-09-18T07:38:30.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:29 smithi083 ceph-mon[63709]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:38:30.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:29 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"} : dispatch 2024-09-18T07:38:30.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:29 smithi096 ceph-mon[49483]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-18T07:38:30.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:29 smithi096 ceph-mon[49483]: Cluster is now healthy 2024-09-18T07:38:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:29 smithi096 ceph-mon[49483]: mds.? [v2:172.21.15.96:6832/824261772,v1:172.21.15.96:6833/824261772] up:boot 2024-09-18T07:38:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:29 smithi096 ceph-mon[49483]: fsmap foofs:1 {0=foofs.smithi083.kjbbry=up:active} 1 up:standby 2024-09-18T07:38:30.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:29 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "mds metadata", "who": "foofs.smithi096.bzilgv"} : dispatch 2024-09-18T07:38:31.064 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:30 smithi096 ceph-mon[49483]: pgmap v332: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 9 op/s 2024-09-18T07:38:31.064 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:30 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:31.064 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:30 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:31.064 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:30 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:31.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:30 smithi083 ceph-mon[63709]: pgmap v332: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 9 op/s 2024-09-18T07:38:31.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:30 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:31.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:30 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:31.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:30 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:32.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:38:32.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:38:33.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:32 smithi083 ceph-mon[63709]: pgmap v333: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-09-18T07:38:33.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:33.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:33.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:32 smithi096 ceph-mon[49483]: pgmap v333: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-09-18T07:38:33.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:33.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:34.023 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:38:34.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:34 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:34.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:34 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:34.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:34 smithi083 ceph-mon[63709]: pgmap v334: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.5 KiB/s wr, 12 op/s 2024-09-18T07:38:34.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:34 smithi083 ceph-mon[63709]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:34.598 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:34 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:34.598 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:34 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:34.598 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:34 smithi096 ceph-mon[49483]: pgmap v334: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.5 KiB/s wr, 12 op/s 2024-09-18T07:38:34.599 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:34 smithi096 ceph-mon[49483]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:34.718 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (9m) 20s ago 19m 20.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (6m) 20s ago 19m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (6m) 2s ago 18m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (9m) 20s ago 19m 54.5M - dad864ee21e9 06ff04a18f50 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (27s) 20s ago 13m 11.7M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2f45d23992ac 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (5s) 2s ago 13m 13.1M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 331a28f7070c 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (9m) 20s ago 20m 524M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (8m) 2s ago 18m 458M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (7m) 20s ago 20m 61.7M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (7m) 2s ago 18m 48.1M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (12m) 20s ago 12m 268M - 4.0 e1d6a67b021e 56f126870417 2024-09-18T07:38:34.719 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (12m) 2s ago 12m 90.7M - 4.0 e1d6a67b021e 58491b1ffc8d 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (9m) 20s ago 18m 10.6M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (9m) 2s ago 18m 11.5M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 20s ago 17m 472M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 20s ago 17m 330M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 20s ago 16m 374M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (3m) 20s ago 16m 429M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (2m) 2s ago 15m 350M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2ca3dcb5baa0 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (2m) 2s ago 15m 337M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 8bd5b8b7b654 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (92s) 2s ago 15m 314M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c3e35aae405e 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (53s) 2s ago 14m 307M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 18c8ec419191 2024-09-18T07:38:34.720 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (8m) 20s ago 18m 56.1M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:38:35.139 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 8 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:38:35.140 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:38:35.141 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 14 2024-09-18T07:38:35.141 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:38:35.141 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:38:35.494 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:38:35.494 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:38:35.494 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "mds", 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "osd", 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "16/23 daemons upgraded", 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading mds daemons", 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:38:35.495 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:38:35.496 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:35 smithi083 ceph-mon[63709]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:35.496 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:35 smithi083 ceph-mon[63709]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:35.496 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:35 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/2044177594' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:35.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:35 smithi096 ceph-mon[49483]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:35.625 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:35 smithi096 ceph-mon[49483]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:35.625 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:35 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/2044177594' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:35.932 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:38:36.300 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:36 smithi096 ceph-mon[49483]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:36.300 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:36 smithi096 ceph-mon[49483]: pgmap v335: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-18T07:38:36.300 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:36 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1711069891' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:38:36.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:36 smithi083 ceph-mon[63709]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:38:36.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:36 smithi083 ceph-mon[63709]: pgmap v335: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-18T07:38:36.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:36 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1711069891' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:38:37.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: Detected new or changed devices on smithi096 2024-09-18T07:38:37.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all mds 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi083.kjbbry"} : dispatch 2024-09-18T07:38:37.619 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi083.kjbbry"}]': finished 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi096.bzilgv"} : dispatch 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi096.bzilgv"}]': finished 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all rgw 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all rbd-mirror 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all ceph-exporter 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.620 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all iscsi 2024-09-18T07:38:37.621 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:37 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: Detected new or changed devices on smithi096 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all mds 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi083.kjbbry"} : dispatch 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi083.kjbbry"}]': finished 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi096.bzilgv"} : dispatch 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi096.bzilgv"}]': finished 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all rgw 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all rbd-mirror 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.623 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all ceph-exporter 2024-09-18T07:38:37.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:37.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:37.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all iscsi 2024-09-18T07:38:37.624 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:37 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:38 smithi083 ceph-mon[63709]: pgmap v336: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.2 KiB/s wr, 10 op/s 2024-09-18T07:38:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-18T07:38:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-18T07:38:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-18T07:38:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-18T07:38:38.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:38 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-18T07:38:38.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:38 smithi096 ceph-mon[49483]: pgmap v336: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.2 KiB/s wr, 10 op/s 2024-09-18T07:38:38.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:38.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-18T07:38:38.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-18T07:38:38.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-18T07:38:38.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:38.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-18T07:38:38.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:38 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-18T07:38:39.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:39 smithi083 ceph-mon[63709]: Upgrade: Updating nfs.foo.0.0.smithi083.jpbwtk (1/2) 2024-09-18T07:38:39.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:39 smithi083 ceph-mon[63709]: Creating key for client.nfs.foo.0.0.smithi083.jpbwtk 2024-09-18T07:38:39.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:39 smithi083 ceph-mon[63709]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-18T07:38:39.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:39 smithi083 ceph-mon[63709]: Rados config object exists: conf-nfs.foo 2024-09-18T07:38:39.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:39 smithi083 ceph-mon[63709]: Creating key for client.nfs.foo.0.0.smithi083.jpbwtk-rgw 2024-09-18T07:38:39.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:39 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-18T07:38:39.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:39 smithi083 ceph-mon[63709]: Bind address in nfs.foo.0.0.smithi083.jpbwtk's ganesha conf is defaulting to empty 2024-09-18T07:38:39.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:39 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:39.369 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:39 smithi083 ceph-mon[63709]: Deploying daemon nfs.foo.0.0.smithi083.jpbwtk on smithi083 2024-09-18T07:38:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:39 smithi096 ceph-mon[49483]: Upgrade: Updating nfs.foo.0.0.smithi083.jpbwtk (1/2) 2024-09-18T07:38:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:39 smithi096 ceph-mon[49483]: Creating key for client.nfs.foo.0.0.smithi083.jpbwtk 2024-09-18T07:38:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:39 smithi096 ceph-mon[49483]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-18T07:38:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:39 smithi096 ceph-mon[49483]: Rados config object exists: conf-nfs.foo 2024-09-18T07:38:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:39 smithi096 ceph-mon[49483]: Creating key for client.nfs.foo.0.0.smithi083.jpbwtk-rgw 2024-09-18T07:38:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:39 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi083.jpbwtk-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-18T07:38:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:39 smithi096 ceph-mon[49483]: Bind address in nfs.foo.0.0.smithi083.jpbwtk's ganesha conf is defaulting to empty 2024-09-18T07:38:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:39 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:39 smithi096 ceph-mon[49483]: Deploying daemon nfs.foo.0.0.smithi083.jpbwtk on smithi083 2024-09-18T07:38:40.358 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:40 smithi083 ceph-mon[63709]: pgmap v337: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-18T07:38:40.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:40 smithi096 ceph-mon[49483]: pgmap v337: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-18T07:38:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:42 smithi083 ceph-mon[63709]: pgmap v338: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 7.6 KiB/s wr, 5 op/s 2024-09-18T07:38:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:42 smithi083 ceph-mon[63709]: Evicting (and blocklisting) client session 14562 (172.21.15.83:0/473556564) 2024-09-18T07:38:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:42 smithi083 ceph-mon[63709]: from='mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617]' entity='mds.foofs.smithi083.kjbbry' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.83:0/473556564"} : dispatch 2024-09-18T07:38:43.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:43.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:42 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:42 smithi096 ceph-mon[49483]: pgmap v338: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 7.6 KiB/s wr, 5 op/s 2024-09-18T07:38:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:42 smithi096 ceph-mon[49483]: Evicting (and blocklisting) client session 14562 (172.21.15.83:0/473556564) 2024-09-18T07:38:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:42 smithi096 ceph-mon[49483]: from='mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617]' entity='mds.foofs.smithi083.kjbbry' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.83:0/473556564"} : dispatch 2024-09-18T07:38:43.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:43.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:42 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:43.982 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617]' entity='mds.foofs.smithi083.kjbbry' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.83:0/473556564"}]': finished 2024-09-18T07:38:43.982 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: osdmap e171: 8 total, 8 up, 8 in 2024-09-18T07:38:43.982 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:43.982 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-18T07:38:43.983 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-18T07:38:43.983 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-18T07:38:43.983 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:43.983 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-18T07:38:43.983 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-18T07:38:43.983 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-18T07:38:43.983 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:43 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617]' entity='mds.foofs.smithi083.kjbbry' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.83:0/473556564"}]': finished 2024-09-18T07:38:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: osdmap e171: 8 total, 8 up, 8 in 2024-09-18T07:38:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-18T07:38:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-18T07:38:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-18T07:38:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:44.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-18T07:38:44.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-18T07:38:44.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi096.vijuth-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-18T07:38:44.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:43 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:45.011 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:44 smithi096 ceph-mon[49483]: Upgrade: Updating nfs.foo.1.0.smithi096.vijuth (2/2) 2024-09-18T07:38:45.012 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:44 smithi096 ceph-mon[49483]: Creating key for client.nfs.foo.1.0.smithi096.vijuth 2024-09-18T07:38:45.012 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:44 smithi096 ceph-mon[49483]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-18T07:38:45.012 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:44 smithi096 ceph-mon[49483]: Rados config object exists: conf-nfs.foo 2024-09-18T07:38:45.012 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:44 smithi096 ceph-mon[49483]: Creating key for client.nfs.foo.1.0.smithi096.vijuth-rgw 2024-09-18T07:38:45.012 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:44 smithi096 ceph-mon[49483]: Bind address in nfs.foo.1.0.smithi096.vijuth's ganesha conf is defaulting to empty 2024-09-18T07:38:45.012 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:44 smithi096 ceph-mon[49483]: Deploying daemon nfs.foo.1.0.smithi096.vijuth on smithi096 2024-09-18T07:38:45.012 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:44 smithi096 ceph-mon[49483]: pgmap v340: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 716 B/s wr, 2 op/s 2024-09-18T07:38:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:44 smithi083 ceph-mon[63709]: Upgrade: Updating nfs.foo.1.0.smithi096.vijuth (2/2) 2024-09-18T07:38:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:44 smithi083 ceph-mon[63709]: Creating key for client.nfs.foo.1.0.smithi096.vijuth 2024-09-18T07:38:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:44 smithi083 ceph-mon[63709]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-18T07:38:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:44 smithi083 ceph-mon[63709]: Rados config object exists: conf-nfs.foo 2024-09-18T07:38:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:44 smithi083 ceph-mon[63709]: Creating key for client.nfs.foo.1.0.smithi096.vijuth-rgw 2024-09-18T07:38:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:44 smithi083 ceph-mon[63709]: Bind address in nfs.foo.1.0.smithi096.vijuth's ganesha conf is defaulting to empty 2024-09-18T07:38:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:44 smithi083 ceph-mon[63709]: Deploying daemon nfs.foo.1.0.smithi096.vijuth on smithi096 2024-09-18T07:38:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:44 smithi083 ceph-mon[63709]: pgmap v340: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 716 B/s wr, 2 op/s 2024-09-18T07:38:46.996 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:46 smithi096 ceph-mon[49483]: pgmap v341: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 716 B/s wr, 2 op/s 2024-09-18T07:38:46.996 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:38:47.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:46 smithi083 ceph-mon[63709]: pgmap v341: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 716 B/s wr, 2 op/s 2024-09-18T07:38:47.042 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:38:47.939 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:47 smithi096 ceph-mon[49483]: from='mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617]' entity='mds.foofs.smithi083.kjbbry' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.96:0/3655200669"} : dispatch 2024-09-18T07:38:47.939 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:47 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:47.939 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:47 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:47.939 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:47 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:48.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:47 smithi083 ceph-mon[63709]: from='mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617]' entity='mds.foofs.smithi083.kjbbry' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.96:0/3655200669"} : dispatch 2024-09-18T07:38:48.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:47 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:48.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:47 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:48.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:47 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:48 smithi096 ceph-mon[49483]: Evicting (and blocklisting) client session 24337 (172.21.15.96:0/3655200669) 2024-09-18T07:38:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:48 smithi096 ceph-mon[49483]: pgmap v342: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-18T07:38:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:48 smithi096 ceph-mon[49483]: from='mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617]' entity='mds.foofs.smithi083.kjbbry' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.96:0/3655200669"}]': finished 2024-09-18T07:38:48.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:48 smithi096 ceph-mon[49483]: osdmap e172: 8 total, 8 up, 8 in 2024-09-18T07:38:49.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:48 smithi083 ceph-mon[63709]: Evicting (and blocklisting) client session 24337 (172.21.15.96:0/3655200669) 2024-09-18T07:38:49.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:48 smithi083 ceph-mon[63709]: pgmap v342: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-18T07:38:49.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:48 smithi083 ceph-mon[63709]: from='mds.? [v2:172.21.15.83:6834/2557412617,v1:172.21.15.83:6835/2557412617]' entity='mds.foofs.smithi083.kjbbry' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.96:0/3655200669"}]': finished 2024-09-18T07:38:49.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:48 smithi083 ceph-mon[63709]: osdmap e172: 8 total, 8 up, 8 in 2024-09-18T07:38:51.261 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:50 smithi096 ceph-mon[49483]: pgmap v344: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-18T07:38:51.261 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:51.261 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:51.261 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:51.261 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:50 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:51.339 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:50 smithi083 ceph-mon[63709]: pgmap v344: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-18T07:38:51.340 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:51.340 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:51.340 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:51.340 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:50 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:52.861 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:52.861 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:52.861 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:52 smithi083 ceph-mon[63709]: pgmap v345: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-18T07:38:52.862 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:52.862 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:52 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:52.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:52.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:52.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:52 smithi096 ceph-mon[49483]: pgmap v345: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-18T07:38:52.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:52.871 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:52 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:54.857 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:54.857 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:54 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:54.857 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:54 smithi083 ceph-mon[63709]: pgmap v346: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-09-18T07:38:54.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:54.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:54 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:54.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:54 smithi096 ceph-mon[49483]: pgmap v346: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-09-18T07:38:57.048 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: pgmap v347: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.049 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all nfs 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi083.jpbwtk"} : dispatch 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi083.jpbwtk"}]': finished 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi096.vijuth"} : dispatch 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi096.vijuth"}]': finished 2024-09-18T07:38:57.050 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: Upgrade: Setting container_image for all nvmeof 2024-09-18T07:38:57.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.051 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:56 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: pgmap v347: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.5 KiB/s wr, 4 op/s 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.052 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all nfs 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.053 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi083.jpbwtk"} : dispatch 2024-09-18T07:38:57.054 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi083.jpbwtk"}]': finished 2024-09-18T07:38:57.054 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi096.vijuth"} : dispatch 2024-09-18T07:38:57.054 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi096.vijuth"}]': finished 2024-09-18T07:38:57.054 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.054 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: Upgrade: Setting container_image for all nvmeof 2024-09-18T07:38:57.054 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:38:57.054 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.054 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:57.054 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:56 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:38:59.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:58 smithi083 ceph-mon[63709]: pgmap v348: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 819 B/s wr, 2 op/s 2024-09-18T07:38:59.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:58 smithi083 ceph-mon[63709]: Upgrade: Updating grafana.smithi083 2024-09-18T07:38:59.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:38:58 smithi083 ceph-mon[63709]: Deploying daemon grafana.smithi083 on smithi083 2024-09-18T07:38:59.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:58 smithi096 ceph-mon[49483]: pgmap v348: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 819 B/s wr, 2 op/s 2024-09-18T07:38:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:58 smithi096 ceph-mon[49483]: Upgrade: Updating grafana.smithi083 2024-09-18T07:38:59.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:38:58 smithi096 ceph-mon[49483]: Deploying daemon grafana.smithi083 on smithi083 2024-09-18T07:39:01.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:00 smithi096 ceph-mon[49483]: pgmap v349: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 686 B/s wr, 2 op/s 2024-09-18T07:39:01.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:00 smithi083 ceph-mon[63709]: pgmap v349: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 686 B/s wr, 2 op/s 2024-09-18T07:39:02.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:39:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:39:03.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:02 smithi083 ceph-mon[63709]: pgmap v350: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-18T07:39:03.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:02 smithi096 ceph-mon[49483]: pgmap v350: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-18T07:39:05.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:04 smithi083 ceph-mon[63709]: pgmap v351: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:05.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:04 smithi096 ceph-mon[49483]: pgmap v351: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:06.309 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:39:07.019 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:39:07.019 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (10m) 15s ago 19m 20.9M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:39:07.019 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (7m) 15s ago 19m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:39:07.019 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (7m) 17s ago 19m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (10m) 15s ago 19m 55.5M - dad864ee21e9 06ff04a18f50 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (59s) 15s ago 13m 30.9M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2f45d23992ac 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (37s) 17s ago 13m 13.2M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 331a28f7070c 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (9m) 15s ago 20m 526M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (8m) 17s ago 19m 458M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (8m) 15s ago 21m 63.9M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (8m) 17s ago 19m 49.3M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (25s) 15s ago 13m 62.3M - 5.9 b22e1e37ddf1 feca655c2070 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (20s) 17s ago 13m 26.9M - 5.9 b22e1e37ddf1 bb79d7945fb7 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (10m) 15s ago 19m 11.8M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (9m) 17s ago 18m 11.9M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (6m) 15s ago 18m 472M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (5m) 15s ago 17m 332M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (4m) 15s ago 17m 375M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:39:07.020 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 15s ago 16m 430M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:39:07.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (3m) 17s ago 16m 350M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2ca3dcb5baa0 2024-09-18T07:39:07.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (2m) 17s ago 16m 338M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 8bd5b8b7b654 2024-09-18T07:39:07.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (2m) 17s ago 15m 314M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c3e35aae405e 2024-09-18T07:39:07.021 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (85s) 17s ago 15m 308M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 18c8ec419191 2024-09-18T07:39:07.021 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (8m) 15s ago 18m 56.1M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:39:07.021 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:06 smithi083 ceph-mon[63709]: pgmap v352: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:07.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:06 smithi096 ceph-mon[49483]: pgmap v352: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:07.457 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 8 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:39:07.458 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:39:07.459 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:39:07.459 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 14 2024-09-18T07:39:07.459 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:39:07.459 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:39:07.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:07 smithi083 ceph-mon[63709]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:07.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:07 smithi083 ceph-mon[63709]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:07.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:07 smithi083 ceph-mon[63709]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:07.727 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:07 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/4144278639' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:07.826 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:39:07.826 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", 2024-09-18T07:39:07.826 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": true, 2024-09-18T07:39:07.826 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [ 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "mds", 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "crash", 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "osd", 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "nfs", 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr", 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "mon" 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": "18/23 daemons upgraded", 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "Currently upgrading grafana daemons", 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:39:07.827 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:39:08.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:07 smithi096 ceph-mon[49483]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:08.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:07 smithi096 ceph-mon[49483]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:08.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:07 smithi096 ceph-mon[49483]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:08.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:07 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/4144278639' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:08.260 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:39:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:08 smithi083 ceph-mon[63709]: pgmap v353: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:08 smithi083 ceph-mon[63709]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:08 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/652462434' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:39:09.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:08 smithi096 ceph-mon[49483]: pgmap v353: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:09.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:08 smithi096 ceph-mon[49483]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:09.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:08 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/652462434' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:39:11.098 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:10 smithi083 ceph-mon[63709]: pgmap v354: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:11.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:10 smithi096 ceph-mon[49483]: pgmap v354: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:12.948 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:12 smithi083 ceph-mon[63709]: pgmap v355: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 938 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:12.948 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:12.948 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:12.948 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:12 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:39:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:12 smithi096 ceph-mon[49483]: pgmap v355: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 938 B/s rd, 341 B/s wr, 1 op/s 2024-09-18T07:39:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:13.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:12 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:39:14.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:14 smithi083 ceph-mon[63709]: pgmap v356: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:15.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:14 smithi096 ceph-mon[49483]: pgmap v356: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:17.060 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:16 smithi083 ceph-mon[63709]: pgmap v357: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:17.060 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:39:17.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:16 smithi096 ceph-mon[49483]: pgmap v357: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:17.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:39:19.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:18 smithi083 ceph-mon[63709]: pgmap v358: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:19.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:18 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:19.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:18 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:19.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:18 smithi096 ceph-mon[49483]: pgmap v358: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:19.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:18 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:19.123 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:18 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:20.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:20 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:20.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:20 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:20.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:20 smithi083 ceph-mon[63709]: pgmap v359: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:20.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:20 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:20.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:20 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:20.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:20 smithi096 ceph-mon[49483]: pgmap v359: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:23.017 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:22 smithi083 ceph-mon[63709]: pgmap v360: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:39:23.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:22 smithi096 ceph-mon[49483]: pgmap v360: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:39:24.812 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:24 smithi083 ceph-mon[63709]: pgmap v361: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:25.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:24 smithi096 ceph-mon[49483]: pgmap v361: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:27.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: pgmap v362: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:27.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:39:27.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:39:27.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-18T07:39:27.119 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.120 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-18T07:39:27.121 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-18T07:39:27.122 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-18T07:39:27.123 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-18T07:39:27.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:39:27.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:39:27.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:39:27.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.124 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:26 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.124 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: pgmap v362: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.125 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.126 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:27.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-18T07:39:27.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-18T07:39:27.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.127 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-18T07:39:27.128 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-18T07:39:27.129 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-18T07:39:27.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:39:27.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:39:27.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:39:27.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:27.130 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:26 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:27 smithi083 ceph-mon[63709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:39:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:27 smithi083 ceph-mon[63709]: Upgrade: Finalizing container_image settings 2024-09-18T07:39:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:27 smithi083 ceph-mon[63709]: Upgrade: Complete! 2024-09-18T07:39:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:39:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:39:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:39:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:28.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:27 smithi096 ceph-mon[49483]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-18T07:39:28.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:27 smithi096 ceph-mon[49483]: Upgrade: Finalizing container_image settings 2024-09-18T07:39:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:27 smithi096 ceph-mon[49483]: Upgrade: Complete! 2024-09-18T07:39:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:39:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:39:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:39:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:29.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:28 smithi083 ceph-mon[63709]: pgmap v363: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:29.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:28 smithi096 ceph-mon[49483]: pgmap v363: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:31.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:30 smithi083 ceph-mon[63709]: pgmap v364: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:31.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:30 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:39:31.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:30 smithi096 ceph-mon[49483]: pgmap v364: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:31.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:30 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:39:32.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:32 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:32.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:32 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:39:33.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:33 smithi083 ceph-mon[63709]: pgmap v365: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:39:33.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:33 smithi096 ceph-mon[49483]: pgmap v365: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:39:35.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:35 smithi096 ceph-mon[49483]: pgmap v366: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:35.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:35 smithi083 ceph-mon[63709]: pgmap v366: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:37.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:37 smithi096 ceph-mon[49483]: pgmap v367: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:37.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:37 smithi083 ceph-mon[63709]: pgmap v367: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:38.618 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:38 smithi083 ceph-mon[63709]: pgmap v368: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:38.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:38 smithi096 ceph-mon[49483]: pgmap v368: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:39.170 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph orch ps' 2024-09-18T07:39:39.473 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:39 smithi083 ceph-mon[63709]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:39.622 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:39 smithi096 ceph-mon[49483]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:40.525 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:40 smithi083 ceph-mon[63709]: pgmap v369: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:40.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:40 smithi096 ceph-mon[49483]: pgmap v369: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:41.087 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:39:41.087 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (11m) 22s ago 20m 21.0M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:39:41.087 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (7m) 22s ago 20m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:39:41.087 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (7m) 51s ago 19m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:39:41.087 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (29s) 22s ago 20m 38.1M - 10.4.0 c8b91775d855 1e1b575a19a3 2024-09-18T07:39:41.087 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (93s) 22s ago 14m 31.2M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2f45d23992ac 2024-09-18T07:39:41.087 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (71s) 51s ago 14m 13.2M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 331a28f7070c 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (10m) 22s ago 21m 534M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (9m) 51s ago 19m 458M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (9m) 22s ago 21m 64.9M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (8m) 51s ago 19m 49.3M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (59s) 22s ago 14m 63.1M - 5.9 b22e1e37ddf1 feca655c2070 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (54s) 51s ago 13m 26.9M - 5.9 b22e1e37ddf1 bb79d7945fb7 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (10m) 22s ago 20m 11.5M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (10m) 51s ago 19m 11.9M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (7m) 22s ago 18m 473M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (6m) 22s ago 18m 333M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (5m) 22s ago 17m 374M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (4m) 22s ago 17m 431M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:39:41.088 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (3m) 51s ago 17m 350M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2ca3dcb5baa0 2024-09-18T07:39:41.089 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (3m) 51s ago 16m 338M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 8bd5b8b7b654 2024-09-18T07:39:41.089 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (2m) 51s ago 16m 314M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c3e35aae405e 2024-09-18T07:39:41.089 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (119s) 51s ago 15m 308M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 18c8ec419191 2024-09-18T07:39:41.089 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (9m) 22s ago 19m 56.1M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:39:41.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:41 smithi083 ceph-mon[63709]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:41.621 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:41 smithi096 ceph-mon[49483]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:39:41.806 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph versions' 2024-09-18T07:39:42.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:42 smithi083 ceph-mon[63709]: pgmap v370: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:39:42.523 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:42 smithi096 ceph-mon[49483]: pgmap v370: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:39:43.826 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 8 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:39:43.827 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:39:43.828 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:39:43.828 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 14 2024-09-18T07:39:43.828 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:39:43.828 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:39:44.581 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-18T07:39:44.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:44 smithi083 ceph-mon[63709]: pgmap v371: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:44.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:44 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/2712726907' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:45.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:44 smithi096 ceph-mon[49483]: pgmap v371: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:45.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:44 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/2712726907' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:39:46.209 INFO:teuthology.orchestra.run.smithi083.stdout:wait for servicemap items w/ changing names to refresh 2024-09-18T07:39:46.855 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:46 smithi083 ceph-mon[63709]: pgmap v372: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:46.855 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:39:46.877 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'sleep 60' 2024-09-18T07:39:47.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:46 smithi096 ceph-mon[49483]: pgmap v372: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:47.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:39:48.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:48 smithi083 ceph-mon[63709]: pgmap v373: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:49.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:48 smithi096 ceph-mon[49483]: pgmap v373: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:51.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:50 smithi083 ceph-mon[63709]: pgmap v374: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:51.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:50 smithi096 ceph-mon[49483]: pgmap v374: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:53.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:52 smithi083 ceph-mon[63709]: pgmap v375: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:39:53.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:52 smithi096 ceph-mon[49483]: pgmap v375: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:39:55.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:54 smithi083 ceph-mon[63709]: pgmap v376: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:55.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:54 smithi096 ceph-mon[49483]: pgmap v376: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:57.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:56 smithi083 ceph-mon[63709]: pgmap v377: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:57.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:56 smithi096 ceph-mon[49483]: pgmap v377: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:58.872 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:39:58 smithi096 ceph-mon[49483]: pgmap v378: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:39:59.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:39:58 smithi083 ceph-mon[63709]: pgmap v378: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:01.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:00 smithi083 ceph-mon[63709]: pgmap v379: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:01.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:00 smithi083 ceph-mon[63709]: overall HEALTH_OK 2024-09-18T07:40:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:00 smithi096 ceph-mon[49483]: pgmap v379: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:01.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:00 smithi096 ceph-mon[49483]: overall HEALTH_OK 2024-09-18T07:40:02.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:40:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:40:03.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:02 smithi083 ceph-mon[63709]: pgmap v380: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:03.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:02 smithi096 ceph-mon[49483]: pgmap v380: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:05.088 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:04 smithi083 ceph-mon[63709]: pgmap v381: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:05.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:04 smithi096 ceph-mon[49483]: pgmap v381: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:07.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:06 smithi083 ceph-mon[63709]: pgmap v382: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:07.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:06 smithi096 ceph-mon[49483]: pgmap v382: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:08 smithi083 ceph-mon[63709]: pgmap v383: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:09.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:08 smithi096 ceph-mon[49483]: pgmap v383: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:11.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:10 smithi083 ceph-mon[63709]: pgmap v384: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:11.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:10 smithi096 ceph-mon[49483]: pgmap v384: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:13.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:12 smithi083 ceph-mon[63709]: pgmap v385: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:13.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:12 smithi096 ceph-mon[49483]: pgmap v385: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:15.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:14 smithi083 ceph-mon[63709]: pgmap v386: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:15.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:14 smithi096 ceph-mon[49483]: pgmap v386: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:17.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:16 smithi083 ceph-mon[63709]: pgmap v387: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:17.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:40:17.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:16 smithi096 ceph-mon[49483]: pgmap v387: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:17.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:40:19.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:18 smithi083 ceph-mon[63709]: pgmap v388: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:19.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:18 smithi096 ceph-mon[49483]: pgmap v388: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:21.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:20 smithi083 ceph-mon[63709]: pgmap v389: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:21.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:20 smithi096 ceph-mon[49483]: pgmap v389: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:23.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:22 smithi083 ceph-mon[63709]: pgmap v390: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:23.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:22 smithi096 ceph-mon[49483]: pgmap v390: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:25.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:24 smithi083 ceph-mon[63709]: pgmap v391: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:25.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:24 smithi096 ceph-mon[49483]: pgmap v391: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:27.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:26 smithi083 ceph-mon[63709]: pgmap v392: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:27.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:26 smithi096 ceph-mon[49483]: pgmap v392: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:40:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:28.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:27 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:28.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-18T07:40:28.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:28.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:27 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:29.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:28 smithi083 ceph-mon[63709]: pgmap v393: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:29.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:40:29.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:40:29.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:29.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:28 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:29.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:28 smithi096 ceph-mon[49483]: pgmap v393: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-18T07:40:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-18T07:40:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:29.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:28 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' 2024-09-18T07:40:31.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:30 smithi083 ceph-mon[63709]: pgmap v394: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:31.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:30 smithi096 ceph-mon[49483]: pgmap v394: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:32.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:31 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:40:32.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:31 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:40:33.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:32 smithi083 ceph-mon[63709]: pgmap v395: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:33.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:32 smithi096 ceph-mon[49483]: pgmap v395: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:35.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:34 smithi083 ceph-mon[63709]: pgmap v396: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:35.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:34 smithi096 ceph-mon[49483]: pgmap v396: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:37.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:36 smithi083 ceph-mon[63709]: pgmap v397: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:37.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:36 smithi096 ceph-mon[49483]: pgmap v397: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:39.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:38 smithi083 ceph-mon[63709]: pgmap v398: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:39.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:38 smithi096 ceph-mon[49483]: pgmap v398: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:41.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:40 smithi083 ceph-mon[63709]: pgmap v399: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:41.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:40 smithi096 ceph-mon[49483]: pgmap v399: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:43.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:42 smithi083 ceph-mon[63709]: pgmap v400: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:43.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:42 smithi096 ceph-mon[49483]: pgmap v400: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:45.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:44 smithi083 ceph-mon[63709]: pgmap v401: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:45.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:44 smithi096 ceph-mon[49483]: pgmap v401: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:47.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:46 smithi083 ceph-mon[63709]: pgmap v402: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:47.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:46 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:40:47.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:46 smithi096 ceph-mon[49483]: pgmap v402: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:47.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:46 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:40:48.959 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:48 smithi083 ceph-mon[63709]: pgmap v403: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:49.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:48 smithi096 ceph-mon[49483]: pgmap v403: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:49.344 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph orch ps' 2024-09-18T07:40:50.993 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:50 smithi083 ceph-mon[63709]: pgmap v404: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:51.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:50 smithi096 ceph-mon[49483]: pgmap v404: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:51.246 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-18T07:40:51.247 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.smithi083 smithi083 *:9093,9094 running (12m) 92s ago 21m 21.0M - 0.27.0 11f11916f8cd 83b4996d4b82 2024-09-18T07:40:51.247 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi083 smithi083 running (9m) 92s ago 21m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 a667dd62df61 2024-09-18T07:40:51.247 INFO:teuthology.orchestra.run.smithi083.stdout:crash.smithi096 smithi096 running (8m) 2m ago 20m 6899k - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 bd4719b5a93f 2024-09-18T07:40:51.247 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.smithi083 smithi083 *:3000 running (99s) 92s ago 21m 38.1M - 10.4.0 c8b91775d855 1e1b575a19a3 2024-09-18T07:40:51.247 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi083.kjbbry smithi083 running (2m) 92s ago 15m 31.2M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2f45d23992ac 2024-09-18T07:40:51.247 INFO:teuthology.orchestra.run.smithi083.stdout:mds.foofs.smithi096.bzilgv smithi096 running (2m) 2m ago 15m 13.2M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 331a28f7070c 2024-09-18T07:40:51.247 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi083.vchvnu smithi083 *:8443,9283,8765 running (11m) 92s ago 22m 534M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 557e489b047d 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.smithi096.caifnv smithi096 *:8443,9283,8765 running (10m) 2m ago 20m 458M - 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 abc469f61695 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi083 smithi083 running (10m) 92s ago 22m 64.9M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c72d955a581d 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:mon.smithi096 smithi096 running (9m) 2m ago 20m 49.3M 2048M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 baa2fbd25f23 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.0.0.smithi083.jpbwtk smithi083 *:2049 running (2m) 92s ago 15m 63.1M - 5.9 b22e1e37ddf1 feca655c2070 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:nfs.foo.1.0.smithi096.vijuth smithi096 *:2049 running (2m) 2m ago 15m 26.9M - 5.9 b22e1e37ddf1 bb79d7945fb7 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi083 smithi083 *:9100 running (11m) 92s ago 21m 11.5M - 1.7.0 72c9c2088986 2529726cf368 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.smithi096 smithi096 *:9100 running (11m) 2m ago 20m 11.9M - 1.7.0 72c9c2088986 7fc32a98e854 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 92s ago 19m 473M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 614f60ad38ff 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (7m) 92s ago 19m 333M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c5493455402e 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (6m) 92s ago 19m 374M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 1e82d2ea2dc2 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (5m) 92s ago 18m 431M 1936M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 9e0abb53ec3c 2024-09-18T07:40:51.248 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi096 running (5m) 2m ago 18m 350M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 2ca3dcb5baa0 2024-09-18T07:40:51.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi096 running (4m) 2m ago 17m 338M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 8bd5b8b7b654 2024-09-18T07:40:51.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi096 running (3m) 2m ago 17m 314M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 c3e35aae405e 2024-09-18T07:40:51.249 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi096 running (3m) 2m ago 17m 308M 2704M 19.3.0-4983-ge7fb7b56 b22e1e37ddf1 18c8ec419191 2024-09-18T07:40:51.249 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.smithi083 smithi083 *:9095 running (10m) 92s ago 20m 56.1M - 2.51.0 1d3b7f56885b e68b9f9dcccc 2024-09-18T07:40:51.821 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph orch upgrade status' 2024-09-18T07:40:53.074 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:52 smithi083 ceph-mon[63709]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:40:53.074 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:52 smithi083 ceph-mon[63709]: pgmap v405: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:53.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:52 smithi096 ceph-mon[49483]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:40:53.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:52 smithi096 ceph-mon[49483]: pgmap v405: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:40:53.780 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:40:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "target_image": null, 2024-09-18T07:40:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "in_progress": false, 2024-09-18T07:40:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "which": "", 2024-09-18T07:40:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "services_complete": [], 2024-09-18T07:40:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "progress": null, 2024-09-18T07:40:53.780 INFO:teuthology.orchestra.run.smithi083.stdout: "message": "", 2024-09-18T07:40:53.781 INFO:teuthology.orchestra.run.smithi083.stdout: "is_paused": false 2024-09-18T07:40:53.781 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:40:54.486 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph health detail' 2024-09-18T07:40:55.098 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:54 smithi083 ceph-mon[63709]: pgmap v406: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:55.098 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:54 smithi083 ceph-mon[63709]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:40:55.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:54 smithi096 ceph-mon[49483]: pgmap v406: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:55.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:54 smithi096 ceph-mon[49483]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:40:56.549 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-09-18T07:40:57.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:56 smithi083 ceph-mon[63709]: pgmap v407: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:57.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:56 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1073932447' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:40:57.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:56 smithi096 ceph-mon[49483]: pgmap v407: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:57.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:56 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1073932447' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-18T07:40:57.215 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph versions' 2024-09-18T07:40:59.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:58 smithi083 ceph-mon[63709]: pgmap v408: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:59.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:58 smithi096 ceph-mon[49483]: pgmap v408: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:40:59.248 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 8 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": { 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 2 2024-09-18T07:40:59.249 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:40:59.250 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-09-18T07:40:59.250 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)": 14 2024-09-18T07:40:59.250 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:40:59.250 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-09-18T07:40:59.868 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:40:59 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1762501861' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:40:59.914 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-18T07:41:00.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:40:59 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1762501861' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:41:01.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:00 smithi083 ceph-mon[63709]: pgmap v409: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:01.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:00 smithi096 ceph-mon[49483]: pgmap v409: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:02.075 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-09-18T07:41:02.117 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:01 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:41:02.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:01 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:41:02.637 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -e sha1=e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-18T07:41:02.930 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:02 smithi083 ceph-mon[63709]: pgmap v410: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:41:02.931 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:02 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/1357634653' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:41:03.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:02 smithi096 ceph-mon[49483]: pgmap v410: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:41:03.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:02 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/1357634653' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:41:04.702 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.3.0-4983-ge7fb7b56 (e7fb7b56dbae911c82c9a0310bb8cf8c37e8363e) squid (dev)" 2024-09-18T07:41:04.985 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:04 smithi083 ceph-mon[63709]: pgmap v411: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:04.985 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:04 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/773265527' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:41:05.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:04 smithi096 ceph-mon[49483]: pgmap v411: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:05.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:04 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/773265527' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-18T07:41:05.284 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-18T07:41:05.285 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-18T07:41:05.285 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-18T07:41:07.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:06 smithi083 ceph-mon[63709]: pgmap v412: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:07.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:06 smithi096 ceph-mon[49483]: pgmap v412: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:07.263 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-09-18T07:41:07.263 INFO:teuthology.orchestra.run.smithi083.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-18T07:18:45.888678Z", "last_refresh": "2024-09-18T07:39:18.548869Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-18T07:18:40.384022Z", "last_refresh": "2024-09-18T07:38:49.976014Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-18T07:18:43.135860Z", "last_refresh": "2024-09-18T07:39:18.549009Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-18T07:25:14.694022Z", "last_refresh": "2024-09-18T07:38:49.976599Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-18T07:18:38.506217Z", "last_refresh": "2024-09-18T07:38:49.976108Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi083:172.21.15.83=smithi083", "smithi096:172.21.15.96=smithi096"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-18T07:19:27.514426Z", "last_refresh": "2024-09-18T07:38:49.976181Z", "running": 2, "size": 2}}, {"events": ["2024-09-18T07:40:27.893283Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-18T07:25:34.621787Z", "last_refresh": "2024-09-18T07:38:49.976665Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-18T07:18:44.228178Z", "last_refresh": "2024-09-18T07:38:49.976251Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9", "last_refresh": "2024-09-18T07:38:49.976319Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-18T07:18:42.006767Z", "last_refresh": "2024-09-18T07:39:18.549156Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-18T07:41:07.823 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-18T07:41:07.824 DEBUG:teuthology.parallel:result is None 2024-09-18T07:41:07.824 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-18T07:41:07.835 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:41:07.835 DEBUG:teuthology.orchestra.run.smithi083:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-18T07:41:07.861 INFO:teuthology.orchestra.run.smithi083.stderr:+ umount /mnt/foo 2024-09-18T07:41:07.893 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-18T07:41:07.903 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:41:07.903 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-18T07:41:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:08 smithi083 ceph-mon[63709]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:41:09.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:08 smithi083 ceph-mon[63709]: pgmap v413: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:09.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:08 smithi096 ceph-mon[49483]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-18T07:41:09.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:08 smithi096 ceph-mon[49483]: pgmap v413: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:09.912 INFO:teuthology.orchestra.run.smithi083.stdout: "foo" 2024-09-18T07:41:10.599 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-18T07:41:11.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:10 smithi083 ceph-mon[63709]: pgmap v414: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:11.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:10 smithi083 ceph-mon[63709]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:41:11.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:10 smithi096 ceph-mon[49483]: pgmap v414: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-18T07:41:11.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:10 smithi096 ceph-mon[49483]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:41:12.531 INFO:teuthology.orchestra.run.smithi083.stdout:[ 2024-09-18T07:41:12.531 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-09-18T07:41:12.531 INFO:teuthology.orchestra.run.smithi083.stdout: "access_type": "RW", 2024-09-18T07:41:12.531 INFO:teuthology.orchestra.run.smithi083.stdout: "clients": [], 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "cluster_id": "foo", 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "export_id": 1, 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "fsal": { 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "fs_name": "foofs", 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "name": "CEPH", 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "user_id": "nfs.foo.1" 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "path": "/", 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "protocols": [ 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: 4 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "pseudo": "/fake", 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "security_label": true, 2024-09-18T07:41:12.532 INFO:teuthology.orchestra.run.smithi083.stdout: "squash": "none", 2024-09-18T07:41:12.533 INFO:teuthology.orchestra.run.smithi083.stdout: "transports": [ 2024-09-18T07:41:12.533 INFO:teuthology.orchestra.run.smithi083.stdout: "TCP" 2024-09-18T07:41:12.533 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-09-18T07:41:12.533 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-09-18T07:41:12.533 INFO:teuthology.orchestra.run.smithi083.stdout:] 2024-09-18T07:41:13.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:12 smithi083 ceph-mon[63709]: pgmap v415: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:41:13.121 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:12 smithi096 ceph-mon[49483]: pgmap v415: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-18T07:41:13.170 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-18T07:41:14.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:13 smithi083 ceph-mon[63709]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:41:14.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:13 smithi096 ceph-mon[49483]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-18T07:41:14.857 INFO:teuthology.orchestra.run.smithi083.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-18T07:41:14.857 INFO:teuthology.orchestra.run.smithi083.stdout:foo grace 2024-09-18T07:41:14.857 INFO:teuthology.orchestra.run.smithi083.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-18T07:41:14.858 INFO:teuthology.orchestra.run.smithi083.stdout:foo conf-nfs.foo 2024-09-18T07:41:14.858 INFO:teuthology.orchestra.run.smithi083.stdout:foo export-1 2024-09-18T07:41:15.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:14 smithi083 ceph-mon[63709]: pgmap v416: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-18T07:41:15.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:14 smithi083 ceph-mon[63709]: mgrmap e44: smithi083.vchvnu(active, since 10m), standbys: smithi096.caifnv 2024-09-18T07:41:15.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:14 smithi096 ceph-mon[49483]: pgmap v416: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-18T07:41:15.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:14 smithi096 ceph-mon[49483]: mgrmap e44: smithi083.vchvnu(active, since 10m), standbys: smithi096.caifnv 2024-09-18T07:41:15.469 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-18T07:41:15.480 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:41:15.481 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 126f471a-758e-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-18T07:41:15.481 DEBUG:teuthology.orchestra.run.smithi083:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-18T07:41:15.481 DEBUG:teuthology.orchestra.run.smithi083:> ' 2024-09-18T07:41:17.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:16 smithi083 ceph-mon[63709]: pgmap v417: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-18T07:41:17.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:16 smithi083 ceph-mon[63709]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:41:17.205 INFO:teuthology.orchestra.run.smithi083.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-18T07:41:17.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:16 smithi096 ceph-mon[49483]: pgmap v417: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-18T07:41:17.372 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:16 smithi096 ceph-mon[49483]: from='mgr.34102 172.21.15.83:0/3913934577' entity='mgr.smithi083.vchvnu' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-18T07:41:17.455 INFO:teuthology.orchestra.run.smithi083.stderr:+ [[ 8 -gt 2 ]] 2024-09-18T07:41:18.021 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:17 smithi083 ceph-mon[63709]: from='client.? 172.21.15.83:0/2863238442' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-18T07:41:18.023 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-18T07:41:18.035 INFO:tasks.cephadm:Teardown begin 2024-09-18T07:41:18.036 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-18T07:41:18.062 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-18T07:41:18.095 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-18T07:41:18.096 DEBUG:teuthology.orchestra.run.smithi083:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-18T07:41:18.118 DEBUG:teuthology.orchestra.run.smithi096:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-18T07:41:18.153 INFO:tasks.cephadm:Stopping all daemons... 2024-09-18T07:41:18.154 INFO:tasks.cephadm.mon.smithi083:Stopping mon.smithi083... 2024-09-18T07:41:18.154 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi083 2024-09-18T07:41:18.368 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:18 smithi083 systemd[1]: Stopping Ceph mon.smithi083 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:41:18.371 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:17 smithi096 ceph-mon[49483]: from='client.? 172.21.15.83:0/2863238442' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-18T07:41:18.759 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:18 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083[63705]: 2024-09-18T07:41:18.451+0000 7fcf6c6f9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi083 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:41:18.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:18 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083[63705]: 2024-09-18T07:41:18.451+0000 7fcf6c6f9640 -1 mon.smithi083@0(leader) e3 *** Got Signal Terminated *** 2024-09-18T07:41:18.760 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:18 smithi083 podman[93501]: 2024-09-18 07:41:18.595176731 +0000 UTC m=+0.256613691 container died c72d955a581d13f3548bd1aef89b01164c895775d5c406482bc9a89b94b7c7df (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-18T07:41:19.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:18 smithi083 podman[93501]: 2024-09-18 07:41:18.952247441 +0000 UTC m=+0.613684402 container cleanup c72d955a581d13f3548bd1aef89b01164c895775d5c406482bc9a89b94b7c7df (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-18T07:41:19.118 INFO:journalctl@ceph.mon.smithi083.smithi083.stdout:Sep 18 07:41:18 smithi083 bash[93501]: ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi083 2024-09-18T07:41:19.429 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi083.service' 2024-09-18T07:41:19.474 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:41:19.475 INFO:tasks.cephadm.mon.smithi083:Stopped mon.smithi083 2024-09-18T07:41:19.475 INFO:tasks.cephadm.mon.smithi096:Stopping mon.smithi096... 2024-09-18T07:41:19.475 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi096 2024-09-18T07:41:19.842 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:19 smithi096 systemd[1]: Stopping Ceph mon.smithi096 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:41:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:19 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096[49479]: 2024-09-18T07:41:19.838+0000 7f7844000640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi096 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:41:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:19 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096[49479]: 2024-09-18T07:41:19.838+0000 7f7844000640 -1 mon.smithi096@1(peon) e3 *** Got Signal Terminated *** 2024-09-18T07:41:20.122 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:20 smithi096 podman[68415]: 2024-09-18 07:41:20.053724995 +0000 UTC m=+0.392885559 container died baa2fbd25f23ddc1711b739464db15e5ef88d4eb81c76625145790deb6af1d95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-18T07:41:20.590 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:20 smithi096 podman[68415]: 2024-09-18 07:41:20.572781165 +0000 UTC m=+0.911941727 container cleanup baa2fbd25f23ddc1711b739464db15e5ef88d4eb81c76625145790deb6af1d95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:41:20.590 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:20 smithi096 bash[68415]: ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096 2024-09-18T07:41:21.047 INFO:journalctl@ceph.mon.smithi096.smithi096.stdout:Sep 18 07:41:20 smithi096 podman[68427]: 2024-09-18 07:41:20.791177455 +0000 UTC m=+0.732231607 container remove baa2fbd25f23ddc1711b739464db15e5ef88d4eb81c76625145790deb6af1d95 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-mon-smithi096, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-18T07:41:21.052 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@mon.smithi096.service' 2024-09-18T07:41:21.099 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:41:21.099 INFO:tasks.cephadm.mon.smithi096:Stopped mon.smithi096 2024-09-18T07:41:21.099 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-18T07:41:21.100 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.0 2024-09-18T07:41:21.401 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:21 smithi083 systemd[1]: Stopping Ceph osd.0 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:41:21.868 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:21 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[68582]: 2024-09-18T07:41:21.396+0000 7f997ce00640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:41:21.868 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:21 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[68582]: 2024-09-18T07:41:21.396+0000 7f997ce00640 -1 osd.0 172 *** Got signal Terminated *** 2024-09-18T07:41:21.868 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:21 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0[68582]: 2024-09-18T07:41:21.396+0000 7f997ce00640 -1 osd.0 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:41:27.267 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:26 smithi083 podman[93635]: 2024-09-18 07:41:26.951201784 +0000 UTC m=+5.656565897 container died 614f60ad38ff05050a86bc0f0112589953102c001376346d656254104a77661f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-18T07:41:27.601 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:27 smithi083 podman[93635]: 2024-09-18 07:41:27.339645931 +0000 UTC m=+6.045010033 container cleanup 614f60ad38ff05050a86bc0f0112589953102c001376346d656254104a77661f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-18T07:41:27.601 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:27 smithi083 bash[93635]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0 2024-09-18T07:41:27.868 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:27 smithi083 podman[93649]: 2024-09-18 07:41:27.599499156 +0000 UTC m=+0.642204423 container remove 614f60ad38ff05050a86bc0f0112589953102c001376346d656254104a77661f (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-18T07:41:28.272 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:27 smithi083 podman[93739]: 2024-09-18 07:41:27.958847119 +0000 UTC m=+0.110096192 container create 860568c58def11f6e3bf9db5160cdb89ab8298cd43fab8177f1e9adc090117f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-18T07:41:28.272 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:27 smithi083 podman[93739]: 2024-09-18 07:41:27.86458757 +0000 UTC m=+0.015836615 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:41:28.273 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:28 smithi083 podman[93739]: 2024-09-18 07:41:28.178287835 +0000 UTC m=+0.329536871 container init 860568c58def11f6e3bf9db5160cdb89ab8298cd43fab8177f1e9adc090117f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-18T07:41:28.273 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:28 smithi083 podman[93739]: 2024-09-18 07:41:28.181502201 +0000 UTC m=+0.332751238 container start 860568c58def11f6e3bf9db5160cdb89ab8298cd43fab8177f1e9adc090117f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-18T07:41:28.273 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:28 smithi083 podman[93739]: 2024-09-18 07:41:28.269854748 +0000 UTC m=+0.421103792 container attach 860568c58def11f6e3bf9db5160cdb89ab8298cd43fab8177f1e9adc090117f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD) 2024-09-18T07:41:28.618 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:28 smithi083 podman[93739]: 2024-09-18 07:41:28.375029847 +0000 UTC m=+0.526278885 container died 860568c58def11f6e3bf9db5160cdb89ab8298cd43fab8177f1e9adc090117f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD) 2024-09-18T07:41:28.935 INFO:journalctl@ceph.osd.0.smithi083.stdout:Sep 18 07:41:28 smithi083 podman[93762]: 2024-09-18 07:41:28.915213466 +0000 UTC m=+0.534330749 container remove 860568c58def11f6e3bf9db5160cdb89ab8298cd43fab8177f1e9adc090117f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:41:28.940 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.0.service' 2024-09-18T07:41:28.977 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:41:28.977 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-18T07:41:28.977 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-18T07:41:28.977 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.1 2024-09-18T07:41:29.305 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:29 smithi083 systemd[1]: Stopping Ceph osd.1 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:41:29.618 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:29 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[70956]: 2024-09-18T07:41:29.301+0000 7f23125cb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:41:29.618 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:29 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[70956]: 2024-09-18T07:41:29.301+0000 7f23125cb640 -1 osd.1 172 *** Got signal Terminated *** 2024-09-18T07:41:29.618 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:29 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1[70956]: 2024-09-18T07:41:29.301+0000 7f23125cb640 -1 osd.1 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:41:35.077 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:34 smithi083 podman[93843]: 2024-09-18 07:41:34.743841629 +0000 UTC m=+5.545453597 container died c5493455402e5b7f26956f72bb01e9566197f7699abff7a9f56570402457c093 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-18T07:41:35.077 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:35 smithi083 podman[93843]: 2024-09-18 07:41:35.075414731 +0000 UTC m=+5.877026690 container cleanup c5493455402e5b7f26956f72bb01e9566197f7699abff7a9f56570402457c093 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-18T07:41:35.368 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:35 smithi083 bash[93843]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1 2024-09-18T07:41:35.368 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:35 smithi083 podman[93858]: 2024-09-18 07:41:35.278398082 +0000 UTC m=+0.537474048 container remove c5493455402e5b7f26956f72bb01e9566197f7699abff7a9f56570402457c093 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-18T07:41:36.017 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:35 smithi083 podman[93949]: 2024-09-18 07:41:35.68758379 +0000 UTC m=+0.104095745 container create 8d0a3a2e0b844565e90ecc01a64cb33dc2f244e093c24a142b4ee21c2fd63112 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:41:36.017 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:35 smithi083 podman[93949]: 2024-09-18 07:41:35.59960322 +0000 UTC m=+0.016115140 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:41:36.017 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:35 smithi083 podman[93949]: 2024-09-18 07:41:35.923157311 +0000 UTC m=+0.339669221 container init 8d0a3a2e0b844565e90ecc01a64cb33dc2f244e093c24a142b4ee21c2fd63112 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-18T07:41:36.017 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:35 smithi083 podman[93949]: 2024-09-18 07:41:35.926723804 +0000 UTC m=+0.343235716 container start 8d0a3a2e0b844565e90ecc01a64cb33dc2f244e093c24a142b4ee21c2fd63112 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:41:36.017 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:36 smithi083 podman[93949]: 2024-09-18 07:41:36.014972063 +0000 UTC m=+0.431483972 container attach 8d0a3a2e0b844565e90ecc01a64cb33dc2f244e093c24a142b4ee21c2fd63112 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-18T07:41:36.368 INFO:journalctl@ceph.osd.1.smithi083.stdout:Sep 18 07:41:36 smithi083 podman[93949]: 2024-09-18 07:41:36.118002731 +0000 UTC m=+0.534514641 container died 8d0a3a2e0b844565e90ecc01a64cb33dc2f244e093c24a142b4ee21c2fd63112 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0) 2024-09-18T07:41:36.739 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.1.service' 2024-09-18T07:41:36.776 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:41:36.776 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-18T07:41:36.777 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-18T07:41:36.777 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.2 2024-09-18T07:41:37.091 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:36 smithi083 systemd[1]: Stopping Ceph osd.2 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:41:37.368 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:37 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[73402]: 2024-09-18T07:41:37.088+0000 7fb9b1aaa640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:41:37.368 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:37 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[73402]: 2024-09-18T07:41:37.088+0000 7fb9b1aaa640 -1 osd.2 172 *** Got signal Terminated *** 2024-09-18T07:41:37.368 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:37 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2[73402]: 2024-09-18T07:41:37.088+0000 7fb9b1aaa640 -1 osd.2 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:41:42.833 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:42 smithi083 podman[94068]: 2024-09-18 07:41:42.561398553 +0000 UTC m=+5.563223092 container died 1e82d2ea2dc2217717490e81957f23d3e80567c7515bbe9a6c3f0fe7b3ca66c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5) 2024-09-18T07:41:43.118 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:42 smithi083 podman[94068]: 2024-09-18 07:41:42.921354838 +0000 UTC m=+5.923179369 container cleanup 1e82d2ea2dc2217717490e81957f23d3e80567c7515bbe9a6c3f0fe7b3ca66c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-18T07:41:43.118 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:42 smithi083 bash[94068]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2 2024-09-18T07:41:43.533 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:43 smithi083 podman[94083]: 2024-09-18 07:41:43.198637612 +0000 UTC m=+0.634228275 container remove 1e82d2ea2dc2217717490e81957f23d3e80567c7515bbe9a6c3f0fe7b3ca66c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:41:43.853 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:43 smithi083 podman[94173]: 2024-09-18 07:41:43.531793357 +0000 UTC m=+0.105713426 container create 93e76e1be3f90dfd3757ec635bf8e3d5b6dd2a00baee3211f29fad6487471518 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:41:43.853 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:43 smithi083 podman[94173]: 2024-09-18 07:41:43.442035619 +0000 UTC m=+0.015955700 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:41:43.853 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:43 smithi083 podman[94173]: 2024-09-18 07:41:43.759626959 +0000 UTC m=+0.333547035 container init 93e76e1be3f90dfd3757ec635bf8e3d5b6dd2a00baee3211f29fad6487471518 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-18T07:41:43.853 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:43 smithi083 podman[94173]: 2024-09-18 07:41:43.762921935 +0000 UTC m=+0.336842009 container start 93e76e1be3f90dfd3757ec635bf8e3d5b6dd2a00baee3211f29fad6487471518 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2) 2024-09-18T07:41:43.853 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:43 smithi083 podman[94173]: 2024-09-18 07:41:43.851246254 +0000 UTC m=+0.425166335 container attach 93e76e1be3f90dfd3757ec635bf8e3d5b6dd2a00baee3211f29fad6487471518 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-18T07:41:44.118 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:43 smithi083 podman[94173]: 2024-09-18 07:41:43.950954239 +0000 UTC m=+0.524874308 container died 93e76e1be3f90dfd3757ec635bf8e3d5b6dd2a00baee3211f29fad6487471518 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-18T07:41:44.505 INFO:journalctl@ceph.osd.2.smithi083.stdout:Sep 18 07:41:44 smithi083 podman[94195]: 2024-09-18 07:41:44.488005358 +0000 UTC m=+0.527198923 container remove 93e76e1be3f90dfd3757ec635bf8e3d5b6dd2a00baee3211f29fad6487471518 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-18T07:41:44.511 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.2.service' 2024-09-18T07:41:44.550 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:41:44.550 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-18T07:41:44.551 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-18T07:41:44.551 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.3 2024-09-18T07:41:44.868 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:44 smithi083 systemd[1]: Stopping Ceph osd.3 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:41:45.251 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:44 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[75858]: 2024-09-18T07:41:44.891+0000 7f9de14bc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:41:45.251 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:44 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[75858]: 2024-09-18T07:41:44.891+0000 7f9de14bc640 -1 osd.3 172 *** Got signal Terminated *** 2024-09-18T07:41:45.251 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:44 smithi083 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3[75858]: 2024-09-18T07:41:44.891+0000 7f9de14bc640 -1 osd.3 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:41:48.622 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:48 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:48.450+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:48.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:48 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:48.347+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:49.434 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:49 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:49.150+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:49.434 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:49 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:49.366+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:49.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:49 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:49.430+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:50.460 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:50 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:50.190+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:50.460 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:50 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:50.382+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:50.618 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:50 smithi083 podman[94274]: 2024-09-18 07:41:50.411497611 +0000 UTC m=+5.643620867 container died 9e0abb53ec3ce72e0ecb373c438a4f7b9993bf2fa4992647943320c0d86aab44 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-18T07:41:50.871 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:50 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:50.457+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:51.044 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:50 smithi083 podman[94274]: 2024-09-18 07:41:50.774066147 +0000 UTC m=+6.006189358 container cleanup 9e0abb53ec3ce72e0ecb373c438a4f7b9993bf2fa4992647943320c0d86aab44 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True) 2024-09-18T07:41:51.044 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:50 smithi083 bash[94274]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3 2024-09-18T07:41:51.368 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:51 smithi083 podman[94289]: 2024-09-18 07:41:51.042388088 +0000 UTC m=+0.621415271 container remove 9e0abb53ec3ce72e0ecb373c438a4f7b9993bf2fa4992647943320c0d86aab44 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2) 2024-09-18T07:41:51.412 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:51 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:51.150+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:51.672 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:51 smithi083 podman[94378]: 2024-09-18 07:41:51.270369071 +0000 UTC m=+0.016367258 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:41:51.673 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:51 smithi083 podman[94378]: 2024-09-18 07:41:51.376620152 +0000 UTC m=+0.122618324 container create 9637c09f81c2b0e6a20ab027a0c24b3661184a2a597fb16e2ad7048963a1af27 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:41:51.673 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:51 smithi083 podman[94378]: 2024-09-18 07:41:51.595919554 +0000 UTC m=+0.341917726 container init 9637c09f81c2b0e6a20ab027a0c24b3661184a2a597fb16e2ad7048963a1af27 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-18T07:41:51.673 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:51 smithi083 podman[94378]: 2024-09-18 07:41:51.599599432 +0000 UTC m=+0.345597604 container start 9637c09f81c2b0e6a20ab027a0c24b3661184a2a597fb16e2ad7048963a1af27 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-18T07:41:51.673 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:51 smithi083 podman[94378]: 2024-09-18 07:41:51.670694921 +0000 UTC m=+0.416693090 container attach 9637c09f81c2b0e6a20ab027a0c24b3661184a2a597fb16e2ad7048963a1af27 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:41:51.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:51 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:51.409+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:51.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:51 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:51.491+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:52.118 INFO:journalctl@ceph.osd.3.smithi083.stdout:Sep 18 07:41:51 smithi083 podman[94378]: 2024-09-18 07:41:51.785910729 +0000 UTC m=+0.531908912 container died 9637c09f81c2b0e6a20ab027a0c24b3661184a2a597fb16e2ad7048963a1af27 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:41:52.397 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.3.service' 2024-09-18T07:41:52.436 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:41:52.437 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-18T07:41:52.437 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-18T07:41:52.437 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.4 2024-09-18T07:41:52.443 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:52 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:52.210+0000 7f52acda6640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:25.306350+0000 front 2024-09-18T07:41:25.306322+0000 (oldest deadline 2024-09-18T07:41:51.206244+0000) 2024-09-18T07:41:52.443 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:52 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:52.135+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:52.725 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:52 smithi096 systemd[1]: Stopping Ceph osd.4 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:41:52.725 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:52 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:52.440+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:52.725 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:52 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:52.479+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:53.122 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:52 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:52.721+0000 7f52b0791640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:41:53.122 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:52 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:52.721+0000 7f52b0791640 -1 osd.4 172 *** Got signal Terminated *** 2024-09-18T07:41:53.122 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:52 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:52.721+0000 7f52b0791640 -1 osd.4 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:41:53.459 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:53 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:53.174+0000 7f52acda6640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:25.306350+0000 front 2024-09-18T07:41:25.306322+0000 (oldest deadline 2024-09-18T07:41:51.206244+0000) 2024-09-18T07:41:53.459 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:53 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:53.174+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:53.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:53 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:53.455+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:53.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:53 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:53.495+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:54.425 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:54 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:54.152+0000 7f52acda6640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:25.306350+0000 front 2024-09-18T07:41:25.306322+0000 (oldest deadline 2024-09-18T07:41:51.206244+0000) 2024-09-18T07:41:54.425 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:54 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:54.137+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:54.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:54 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:54.422+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:54.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:54 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:54.543+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:55.372 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:55 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:55.196+0000 7f52acda6640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:25.306350+0000 front 2024-09-18T07:41:25.306322+0000 (oldest deadline 2024-09-18T07:41:51.206244+0000) 2024-09-18T07:41:55.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:55 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:55.107+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:55.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:55 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:55.551+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:55.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:55 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:55.374+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:55.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:55 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:55.374+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:41:56.413 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:56 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:56.198+0000 7f52acda6640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:25.306350+0000 front 2024-09-18T07:41:25.306322+0000 (oldest deadline 2024-09-18T07:41:51.206244+0000) 2024-09-18T07:41:56.413 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:56 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:56.153+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:56.413 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:56 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:56.153+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:41:56.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:56 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:56.523+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:56.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:56 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:56.410+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:56.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:56 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:56.410+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:41:57.432 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:57 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:57.179+0000 7f52acda6640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:25.306350+0000 front 2024-09-18T07:41:25.306322+0000 (oldest deadline 2024-09-18T07:41:51.206244+0000) 2024-09-18T07:41:57.432 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:57 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4[53394]: 2024-09-18T07:41:57.179+0000 7f52acda6640 -1 osd.4 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.907202+0000 front 2024-09-18T07:41:32.907319+0000 (oldest deadline 2024-09-18T07:41:56.907764+0000) 2024-09-18T07:41:57.433 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:57 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:57.121+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:57.433 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:57 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:57.121+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:41:57.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:57 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:57.429+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:57.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:57 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:57.429+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:41:57.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:57 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:57.559+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:57.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:57 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:57.559+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:41:58.372 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:58 smithi096 podman[68564]: 2024-09-18 07:41:58.174307021 +0000 UTC m=+5.548307903 container died 2ca3dcb5baa0011957d859d007072eff7fb29205ecec4d5bcdac20a7afc3e7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:41:58.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:58.097+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:58.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:58.097+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:41:58.818 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:58 smithi096 podman[68564]: 2024-09-18 07:41:58.623060371 +0000 UTC m=+5.997061252 container cleanup 2ca3dcb5baa0011957d859d007072eff7fb29205ecec4d5bcdac20a7afc3e7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:41:58.818 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:58 smithi096 bash[68564]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4 2024-09-18T07:41:58.818 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:58.519+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:58.819 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:58.519+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:41:58.819 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:58.402+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:58.819 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:58 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:58.402+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:41:59.080 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:58 smithi096 podman[68595]: 2024-09-18 07:41:58.816198394 +0000 UTC m=+0.634913720 container remove 2ca3dcb5baa0011957d859d007072eff7fb29205ecec4d5bcdac20a7afc3e7b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-18T07:41:59.350 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:59 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:59.077+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:41:59.350 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:41:59 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:41:59.077+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:41:59.351 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:59 smithi096 podman[68686]: 2024-09-18 07:41:59.13665808 +0000 UTC m=+0.015663788 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:41:59.351 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:59 smithi096 podman[68686]: 2024-09-18 07:41:59.24196503 +0000 UTC m=+0.120970737 container create 69747b2ecae4a450f10e96e89957ca25b0c2630a7414c8bd1a524777d12c2baa (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-18T07:41:59.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:59 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:59.422+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:41:59.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:41:59 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:41:59.422+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:41:59.622 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:59 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:59.568+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:41:59.622 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:41:59 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:41:59.568+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:41:59.623 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:59 smithi096 podman[68686]: 2024-09-18 07:41:59.461067087 +0000 UTC m=+0.340072794 container init 69747b2ecae4a450f10e96e89957ca25b0c2630a7414c8bd1a524777d12c2baa (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-18T07:41:59.623 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:59 smithi096 podman[68686]: 2024-09-18 07:41:59.464436019 +0000 UTC m=+0.343441735 container start 69747b2ecae4a450f10e96e89957ca25b0c2630a7414c8bd1a524777d12c2baa (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-18T07:41:59.623 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:59 smithi096 podman[68686]: 2024-09-18 07:41:59.53583832 +0000 UTC m=+0.414844042 container attach 69747b2ecae4a450f10e96e89957ca25b0c2630a7414c8bd1a524777d12c2baa (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-18T07:41:59.932 INFO:journalctl@ceph.osd.4.smithi096.stdout:Sep 18 07:41:59 smithi096 podman[68686]: 2024-09-18 07:41:59.651310689 +0000 UTC m=+0.530316405 container died 69747b2ecae4a450f10e96e89957ca25b0c2630a7414c8bd1a524777d12c2baa (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-18T07:42:00.233 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:00.094+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:42:00.234 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:00.094+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:42:00.254 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.4.service' 2024-09-18T07:42:00.334 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:42:00.334 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-18T07:42:00.334 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-18T07:42:00.334 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.5 2024-09-18T07:42:00.490 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:00 smithi096 systemd[1]: Stopping Ceph osd.5 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:42:00.490 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:00.391+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:00.490 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:00.391+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:00.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:00.604+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:00.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:00.604+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:00.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:00.683+0000 7fb7e9d98640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:42:00.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:00.683+0000 7fb7e9d98640 -1 osd.5 172 *** Got signal Terminated *** 2024-09-18T07:42:00.872 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:00 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:00.683+0000 7fb7e9d98640 -1 osd.5 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:42:01.371 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:01.352+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:01.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:01.352+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:01.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:01.114+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:42:01.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:01.114+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:42:01.871 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:01.615+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:01.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:01 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:01.615+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:02.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:02 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:02.321+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:02.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:02 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:02.321+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:02.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:02 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:02.072+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:42:02.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:02 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:02.072+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:42:03.055 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:02 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:02.664+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:03.055 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:02 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:02.664+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:03.318 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:03 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:03.052+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:42:03.318 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:03 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:03.052+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:42:03.621 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:03 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:03.315+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:03.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:03 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:03.315+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:04.069 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:03 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:03.683+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:04.069 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:03 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:03.683+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:04.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:04 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:04.309+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:04.372 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:04 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:04.309+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:04.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:04 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:04.065+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:42:04.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:04 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:04.065+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:42:05.083 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:04 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:04.684+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:05.083 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:04 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:04.684+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:05.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:05 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:05.079+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:24.428394+0000 front 2024-09-18T07:41:24.428339+0000 (oldest deadline 2024-09-18T07:41:48.528206+0000) 2024-09-18T07:42:05.372 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:05 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5[55441]: 2024-09-18T07:42:05.079+0000 7fb7e23a5640 -1 osd.5 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:33.829225+0000 front 2024-09-18T07:41:33.829076+0000 (oldest deadline 2024-09-18T07:41:56.129058+0000) 2024-09-18T07:42:05.373 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:05 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:05.321+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:05.373 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:05 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:05.321+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:06.051 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:05 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:05.716+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:06.051 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:05 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:05.716+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:06.337 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:06 smithi096 podman[68787]: 2024-09-18 07:42:06.048935872 +0000 UTC m=+5.480847996 container died 8bd5b8b7b6540b104a7dee8b1399d9aaa19e0940301d46407c8f9f9ca50d1a85 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-18T07:42:06.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:06 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:06.357+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:06.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:06 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:06.357+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:06.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:06 smithi096 podman[68787]: 2024-09-18 07:42:06.534773642 +0000 UTC m=+5.966685765 container cleanup 8bd5b8b7b6540b104a7dee8b1399d9aaa19e0940301d46407c8f9f9ca50d1a85 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-18T07:42:06.622 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:06 smithi096 bash[68787]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5 2024-09-18T07:42:07.122 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:06 smithi096 podman[68817]: 2024-09-18 07:42:06.794602993 +0000 UTC m=+0.732854137 container remove 8bd5b8b7b6540b104a7dee8b1399d9aaa19e0940301d46407c8f9f9ca50d1a85 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-18T07:42:07.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:06 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:06.701+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:07.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:06 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:06.701+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:07.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:06 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:06.701+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:07.390 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:07 smithi096 podman[68906]: 2024-09-18 07:42:07.028646748 +0000 UTC m=+0.015927392 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:42:07.391 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:07 smithi096 podman[68906]: 2024-09-18 07:42:07.12848416 +0000 UTC m=+0.115764774 container create 2a4e4fc74ee41cc2f0e6539f450f410dc65202d8c67a89623730ad68ee9d3c39 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-18T07:42:07.391 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:07 smithi096 podman[68906]: 2024-09-18 07:42:07.352325115 +0000 UTC m=+0.339605736 container init 2a4e4fc74ee41cc2f0e6539f450f410dc65202d8c67a89623730ad68ee9d3c39 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:42:07.391 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:07 smithi096 podman[68906]: 2024-09-18 07:42:07.355572376 +0000 UTC m=+0.342852999 container start 2a4e4fc74ee41cc2f0e6539f450f410dc65202d8c67a89623730ad68ee9d3c39 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-18T07:42:07.670 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:07 smithi096 podman[68906]: 2024-09-18 07:42:07.444129471 +0000 UTC m=+0.431410103 container attach 2a4e4fc74ee41cc2f0e6539f450f410dc65202d8c67a89623730ad68ee9d3c39 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-18T07:42:07.670 INFO:journalctl@ceph.osd.5.smithi096.stdout:Sep 18 07:42:07 smithi096 podman[68906]: 2024-09-18 07:42:07.554844501 +0000 UTC m=+0.542125123 container died 2a4e4fc74ee41cc2f0e6539f450f410dc65202d8c67a89623730ad68ee9d3c39 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-18T07:42:07.670 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:07 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:07.387+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:07.670 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:07 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:07.387+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:07.670 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:07 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:07.387+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.601012+0000 front 2024-09-18T07:41:41.600960+0000 (oldest deadline 2024-09-18T07:42:06.901116+0000) 2024-09-18T07:42:08.121 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:07 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:07.666+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:08.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:07 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:07.666+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:08.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:07 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:07.666+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:08.163 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.5.service' 2024-09-18T07:42:08.201 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:42:08.201 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-18T07:42:08.201 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-18T07:42:08.201 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.6 2024-09-18T07:42:08.528 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:08 smithi096 systemd[1]: Stopping Ceph osd.6 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:42:08.528 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:08 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:08.367+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:08.528 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:08 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:08.367+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:08.528 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:08 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:08.367+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.601012+0000 front 2024-09-18T07:41:41.600960+0000 (oldest deadline 2024-09-18T07:42:06.901116+0000) 2024-09-18T07:42:08.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:08 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:08.524+0000 7fcefc245640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:42:08.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:08 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:08.524+0000 7fcefc245640 -1 osd.6 172 *** Got signal Terminated *** 2024-09-18T07:42:08.872 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:08 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:08.524+0000 7fcefc245640 -1 osd.6 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:42:08.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:08 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:08.673+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:08.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:08 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:08.673+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:08.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:08 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:08.673+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:09.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:09 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:09.355+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:09.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:09 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:09.355+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:09.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:09 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:09.355+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.601012+0000 front 2024-09-18T07:41:41.600960+0000 (oldest deadline 2024-09-18T07:42:06.901116+0000) 2024-09-18T07:42:10.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:09 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:09.683+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:10.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:09 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:09.683+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:10.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:09 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:09.683+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:10.680 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:10 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:10.391+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:10.680 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:10 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:10.391+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:10.681 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:10 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:10.391+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.601012+0000 front 2024-09-18T07:41:41.600960+0000 (oldest deadline 2024-09-18T07:42:06.901116+0000) 2024-09-18T07:42:10.944 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:10 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:10.677+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:10.944 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:10 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:10.677+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:10.944 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:10 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:10.677+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:11.651 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:11 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:11.388+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:11.651 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:11 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:11.388+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:11.651 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:11 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:11.388+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.601012+0000 front 2024-09-18T07:41:41.600960+0000 (oldest deadline 2024-09-18T07:42:06.901116+0000) 2024-09-18T07:42:12.121 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:11 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:11.649+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:12.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:11 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:11.649+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:12.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:11 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:11.649+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:12.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:11 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:11.649+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:12.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:12 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:12.363+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:12.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:12 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:12.363+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:12.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:12 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:12.363+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.601012+0000 front 2024-09-18T07:41:41.600960+0000 (oldest deadline 2024-09-18T07:42:06.901116+0000) 2024-09-18T07:42:13.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:12 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:12.656+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:13.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:12 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:12.656+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:13.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:12 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:12.656+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:13.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:12 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:12.656+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:13.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:13 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:13.351+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.899114+0000 front 2024-09-18T07:41:22.899011+0000 (oldest deadline 2024-09-18T07:41:48.198803+0000) 2024-09-18T07:42:13.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:13 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:13.351+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:29.299713+0000 front 2024-09-18T07:41:29.299679+0000 (oldest deadline 2024-09-18T07:41:55.199881+0000) 2024-09-18T07:42:13.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:13 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:13.351+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.601012+0000 front 2024-09-18T07:41:41.600960+0000 (oldest deadline 2024-09-18T07:42:06.901116+0000) 2024-09-18T07:42:13.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:13 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6[57478]: 2024-09-18T07:42:13.351+0000 7fcef885a640 -1 osd.6 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:49.702226+0000 front 2024-09-18T07:41:49.702174+0000 (oldest deadline 2024-09-18T07:42:13.202172+0000) 2024-09-18T07:42:13.965 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:13 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:13.665+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:13.966 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:13 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:13.665+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:13.966 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:13 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:13.665+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:13.966 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:13 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:13.665+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:14.246 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:13 smithi096 podman[69009]: 2024-09-18 07:42:13.963732356 +0000 UTC m=+5.543821004 container died c3e35aae405ecbae9d1d3ee3f491c3c1bcb3bba0fd75b2a9bea70b7c4c0af25b (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-18T07:42:14.562 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:14 smithi096 podman[69009]: 2024-09-18 07:42:14.400094075 +0000 UTC m=+5.980182763 container cleanup c3e35aae405ecbae9d1d3ee3f491c3c1bcb3bba0fd75b2a9bea70b7c4c0af25b (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-18T07:42:14.562 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:14 smithi096 bash[69009]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6 2024-09-18T07:42:14.871 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:14 smithi096 podman[69024]: 2024-09-18 07:42:14.55967368 +0000 UTC m=+0.598546429 container remove c3e35aae405ecbae9d1d3ee3f491c3c1bcb3bba0fd75b2a9bea70b7c4c0af25b (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-18T07:42:14.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:14 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:14.678+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:14.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:14 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:14.678+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:14.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:14 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:14.678+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:14.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:14 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:14.678+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:15.256 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:14 smithi096 podman[69110]: 2024-09-18 07:42:14.867283117 +0000 UTC m=+0.015686871 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:42:15.257 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:15 smithi096 podman[69110]: 2024-09-18 07:42:15.002199128 +0000 UTC m=+0.150602872 container create 49bb63029c2a77ad1676f4f2415f2801c4f8b6d759b5db8fcfa6d2dcd21409b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-18T07:42:15.257 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:15 smithi096 podman[69110]: 2024-09-18 07:42:15.254774882 +0000 UTC m=+0.403178633 container init 49bb63029c2a77ad1676f4f2415f2801c4f8b6d759b5db8fcfa6d2dcd21409b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5) 2024-09-18T07:42:15.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:15 smithi096 podman[69110]: 2024-09-18 07:42:15.258438764 +0000 UTC m=+0.406842505 container start 49bb63029c2a77ad1676f4f2415f2801c4f8b6d759b5db8fcfa6d2dcd21409b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-18T07:42:15.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:15 smithi096 podman[69110]: 2024-09-18 07:42:15.337935294 +0000 UTC m=+0.486339044 container attach 49bb63029c2a77ad1676f4f2415f2801c4f8b6d759b5db8fcfa6d2dcd21409b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2) 2024-09-18T07:42:15.622 INFO:journalctl@ceph.osd.6.smithi096.stdout:Sep 18 07:42:15 smithi096 podman[69110]: 2024-09-18 07:42:15.446436198 +0000 UTC m=+0.594839950 container died 49bb63029c2a77ad1676f4f2415f2801c4f8b6d759b5db8fcfa6d2dcd21409b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-18T07:42:16.060 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:15 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:15.707+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:16.060 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:15 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:15.707+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:16.060 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:15 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:15.707+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:16.061 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:15 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:15.707+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:16.083 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.6.service' 2024-09-18T07:42:16.121 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:42:16.121 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-18T07:42:16.121 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-18T07:42:16.121 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.7 2024-09-18T07:42:16.371 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:16 smithi096 systemd[1]: Stopping Ceph osd.7 for 126f471a-758e-11ef-bceb-c7b262605968... 2024-09-18T07:42:16.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:16 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:16.486+0000 7f9ea4752640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-18T07:42:16.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:16 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:16.486+0000 7f9ea4752640 -1 osd.7 172 *** Got signal Terminated *** 2024-09-18T07:42:16.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:16 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:16.486+0000 7f9ea4752640 -1 osd.7 172 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-18T07:42:16.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:16 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:16.721+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:16.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:16 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:16.721+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:16.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:16 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:16.721+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:16.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:16 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:16.721+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:18.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:17 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:17.690+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:18.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:17 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:17.690+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:18.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:17 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:17.690+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:18.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:17 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:17.690+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:19.023 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:18 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:18.664+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:19.023 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:18 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:18.664+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:19.023 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:18 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:18.664+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:19.023 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:18 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:18.664+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:19.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:19 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:19.659+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:19.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:19 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:19.659+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:19.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:19 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:19.659+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:19.872 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:19 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:19.659+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:21.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:20 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:20.626+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-09-18T07:41:22.224705+0000 front 2024-09-18T07:41:22.224726+0000 (oldest deadline 2024-09-18T07:41:48.124466+0000) 2024-09-18T07:42:21.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:20 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:20.626+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-09-18T07:41:32.225534+0000 front 2024-09-18T07:41:32.225460+0000 (oldest deadline 2024-09-18T07:41:57.525360+0000) 2024-09-18T07:42:21.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:20 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:20.626+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-09-18T07:41:41.527085+0000 front 2024-09-18T07:41:41.527069+0000 (oldest deadline 2024-09-18T07:42:06.226792+0000) 2024-09-18T07:42:21.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:20 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:20.626+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-09-18T07:41:48.527722+0000 front 2024-09-18T07:41:48.527663+0000 (oldest deadline 2024-09-18T07:42:11.427520+0000) 2024-09-18T07:42:21.122 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:20 smithi096 ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7[59520]: 2024-09-18T07:42:20.626+0000 7f9e9cd5f640 -1 osd.7 172 heartbeat_check: no reply from 172.21.15.96:6804 osd.4 since back 2024-09-18T07:41:56.128090+0000 front 2024-09-18T07:41:56.128172+0000 (oldest deadline 2024-09-18T07:42:20.128630+0000) 2024-09-18T07:42:22.301 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:22 smithi096 podman[69211]: 2024-09-18 07:42:22.014778321 +0000 UTC m=+5.665688653 container died 18c8ec41919114925a61aea73a7860cd942f40363e6c714f1db599ed13dbf99a (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True) 2024-09-18T07:42:22.621 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:22 smithi096 podman[69211]: 2024-09-18 07:42:22.374439171 +0000 UTC m=+6.025349504 container cleanup 18c8ec41919114925a61aea73a7860cd942f40363e6c714f1db599ed13dbf99a (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, org.label-schema.schema-version=1.0) 2024-09-18T07:42:22.622 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:22 smithi096 bash[69211]: ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7 2024-09-18T07:42:22.989 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:22 smithi096 podman[69254]: 2024-09-18 07:42:22.653029771 +0000 UTC m=+0.634808515 container remove 18c8ec41919114925a61aea73a7860cd942f40363e6c714f1db599ed13dbf99a (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-18T07:42:23.303 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:22 smithi096 podman[69343]: 2024-09-18 07:42:22.889869905 +0000 UTC m=+0.015572567 image pull b22e1e37ddf1ca6e22136cd6540a9ab61e7cafb8bfbc7a91a2f20f7f9941bd9a quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9 2024-09-18T07:42:23.303 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:22 smithi096 podman[69343]: 2024-09-18 07:42:22.996777787 +0000 UTC m=+0.122480474 container create a41c05de57d6af76823e866656be01433172a6651f26cfd5c11e4b9c5a9ca509 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-18T07:42:23.303 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:23 smithi096 podman[69343]: 2024-09-18 07:42:23.218043301 +0000 UTC m=+0.343745957 container init a41c05de57d6af76823e866656be01433172a6651f26cfd5c11e4b9c5a9ca509 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-18T07:42:23.303 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:23 smithi096 podman[69343]: 2024-09-18 07:42:23.221306775 +0000 UTC m=+0.347009428 container start a41c05de57d6af76823e866656be01433172a6651f26cfd5c11e4b9c5a9ca509 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-18T07:42:23.303 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:23 smithi096 podman[69343]: 2024-09-18 07:42:23.301124582 +0000 UTC m=+0.426827244 container attach a41c05de57d6af76823e866656be01433172a6651f26cfd5c11e4b9c5a9ca509 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-18T07:42:23.622 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:23 smithi096 podman[69343]: 2024-09-18 07:42:23.412985786 +0000 UTC m=+0.538688438 container died a41c05de57d6af76823e866656be01433172a6651f26cfd5c11e4b9c5a9ca509 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-18T07:42:24.008 INFO:journalctl@ceph.osd.7.smithi096.stdout:Sep 18 07:42:24 smithi096 podman[69366]: 2024-09-18 07:42:24.004330373 +0000 UTC m=+0.585430004 container remove a41c05de57d6af76823e866656be01433172a6651f26cfd5c11e4b9c5a9ca509 (image=quay.ceph.io/ceph-ci/ceph@sha256:1f5b5a3734f665ef333266f39143ac3ef1eb1951b21258f2307e10332e2d7ac9, name=ceph-126f471a-758e-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-17-1213-e7fb7b5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-18T07:42:24.028 DEBUG:teuthology.orchestra.run.smithi096:> sudo pkill -f 'journalctl -f -n 0 -u ceph-126f471a-758e-11ef-bceb-c7b262605968@osd.7.service' 2024-09-18T07:42:24.068 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-18T07:42:24.068 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-18T07:42:24.068 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 126f471a-758e-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-18T07:45:22.650 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 126f471a-758e-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-18T07:45:42.266 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-18T07:45:42.295 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-18T07:45:42.323 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-18T07:45:42.325 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi/7910284/remote/smithi083/crash 2024-09-18T07:45:42.326 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/crash -- . 2024-09-18T07:45:42.368 INFO:teuthology.orchestra.run.smithi083.stderr:tar: /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-18T07:45:42.368 INFO:teuthology.orchestra.run.smithi083.stderr:tar: Error is not recoverable: exiting now 2024-09-18T07:45:42.370 DEBUG:teuthology.misc:Transferring archived files from smithi096:/var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi/7910284/remote/smithi096/crash 2024-09-18T07:45:42.371 DEBUG:teuthology.orchestra.run.smithi096:> sudo tar c -f - -C /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/crash -- . 2024-09-18T07:45:42.407 INFO:teuthology.orchestra.run.smithi096.stderr:tar: /var/lib/ceph/126f471a-758e-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-18T07:45:42.407 INFO:teuthology.orchestra.run.smithi096.stderr:tar: Error is not recoverable: exiting now 2024-09-18T07:45:42.408 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-18T07:45:42.408 DEBUG:teuthology.orchestra.run.smithi083:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-18T07:45:42.436 INFO:tasks.cephadm:Compressing logs... 2024-09-18T07:45:42.436 DEBUG:teuthology.orchestra.run.smithi083:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-18T07:45:42.479 DEBUG:teuthology.orchestra.run.smithi096:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-18T07:45:42.500 INFO:teuthology.orchestra.run.smithi083.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-18T07:45:42.500 INFO:teuthology.orchestra.run.smithi083.stderr:: No such file or directory 2024-09-18T07:45:42.500 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-18T07:45:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mon.smithi083.log 2024-09-18T07:45:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-18T07:45:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mon.smithi083.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-18T07:45:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-18T07:45:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mgr.smithi083.vchvnu.log 2024-09-18T07:45:42.504 INFO:teuthology.orchestra.run.smithi083.stderr:gzip/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.cephadm.log: -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-18T07:45:42.505 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mgr.smithi083.vchvnu.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.log 2024-09-18T07:45:42.505 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.2.log: 85.7% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-18T07:45:42.512 INFO:teuthology.orchestra.run.smithi096.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-18T07:45:42.512 INFO:teuthology.orchestra.run.smithi096.stderr:: No such file or directory 2024-09-18T07:45:42.512 INFO:teuthology.orchestra.run.smithi096.stderr:gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-18T07:45:42.512 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-18T07:45:42.512 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mgr.smithi096.caifnv.log 2024-09-18T07:45:42.512 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-18T07:45:42.512 INFO:teuthology.orchestra.run.smithi096.stderr:gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi096.bzilgv.log 2024-09-18T07:45:42.512 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mgr.smithi096.caifnv.log: /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-18T07:45:42.513 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi096.bzilgv.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-18T07:45:42.513 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.cephadm.log: 88.4%gzip -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi096.bzilgv.log.gz -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-18T07:45:42.513 INFO:teuthology.orchestra.run.smithi096.stderr: 2024-09-18T07:45:42.513 INFO:teuthology.orchestra.run.smithi096.stderr:gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.log 2024-09-18T07:45:42.513 INFO:teuthology.orchestra.run.smithi096.stderr: 85.7%/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.6.log: /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.5.log: -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-18T07:45:42.513 INFO:teuthology.orchestra.run.smithi096.stderr:gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mon.smithi096.log 2024-09-18T07:45:42.513 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.log: 91.6% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-18T07:45:42.513 INFO:teuthology.orchestra.run.smithi096.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-18T07:45:42.513 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mon.smithi096.log: 87.3% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-18T07:45:42.514 INFO:teuthology.orchestra.run.smithi083.stderr: 91.5% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-18T07:45:42.514 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi083.kjbbry.log 2024-09-18T07:45:42.514 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-18T07:45:42.514 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi083.kjbbry.log: 88.9% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi083.kjbbry.log.gz 2024-09-18T07:45:42.514 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-18T07:45:42.515 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.3.log: 87.3% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-18T07:45:42.515 INFO:teuthology.orchestra.run.smithi083.stderr: 91.2% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-18T07:45:42.515 INFO:teuthology.orchestra.run.smithi096.stderr: 91.3% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-18T07:45:42.528 INFO:teuthology.orchestra.run.smithi096.stderr:/var/log/ceph/cephadm.log: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-18T07:45:42.554 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/cephadm.log: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-18T07:45:42.574 INFO:teuthology.orchestra.run.smithi096.stderr: 89.8% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mgr.smithi096.caifnv.log.gz 2024-09-18T07:45:42.858 INFO:teuthology.orchestra.run.smithi096.stderr: 92.6% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mon.smithi096.log.gz 2024-09-18T07:45:42.879 INFO:teuthology.orchestra.run.smithi083.stderr: 88.7% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mgr.smithi083.vchvnu.log.gz 2024-09-18T07:45:44.425 INFO:teuthology.orchestra.run.smithi083.stderr: 91.8% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-mon.smithi083.log.gz 2024-09-18T07:45:55.069 INFO:teuthology.orchestra.run.smithi083.stderr: 94.4% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-18T07:45:56.790 INFO:teuthology.orchestra.run.smithi096.stderr: 93.9% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-18T07:45:58.633 INFO:teuthology.orchestra.run.smithi096.stderr: 94.2% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-18T07:45:59.594 INFO:teuthology.orchestra.run.smithi083.stderr: 94.1% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-18T07:46:01.116 INFO:teuthology.orchestra.run.smithi083.stderr: 94.2% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-18T07:46:01.572 INFO:teuthology.orchestra.run.smithi083.stderr: 94.1% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-18T07:46:01.574 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-09-18T07:46:01.574 INFO:teuthology.orchestra.run.smithi083.stderr:real 0m19.083s 2024-09-18T07:46:01.574 INFO:teuthology.orchestra.run.smithi083.stderr:user 1m6.956s 2024-09-18T07:46:01.574 INFO:teuthology.orchestra.run.smithi083.stderr:sys 0m2.638s 2024-09-18T07:46:01.650 INFO:teuthology.orchestra.run.smithi096.stderr: 94.2% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-18T07:46:06.307 INFO:teuthology.orchestra.run.smithi096.stderr: 94.1% -- replaced with /var/log/ceph/126f471a-758e-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-18T07:46:06.309 INFO:teuthology.orchestra.run.smithi096.stderr: 2024-09-18T07:46:06.309 INFO:teuthology.orchestra.run.smithi096.stderr:real 0m23.815s 2024-09-18T07:46:06.309 INFO:teuthology.orchestra.run.smithi096.stderr:user 1m10.851s 2024-09-18T07:46:06.309 INFO:teuthology.orchestra.run.smithi096.stderr:sys 0m2.826s 2024-09-18T07:46:06.309 INFO:tasks.cephadm:Archiving logs... 2024-09-18T07:46:06.310 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi/7910284/remote/smithi083/log 2024-09-18T07:46:06.311 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-18T07:46:15.614 DEBUG:teuthology.misc:Transferring archived files from smithi096:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi/7910284/remote/smithi096/log 2024-09-18T07:46:15.615 DEBUG:teuthology.orchestra.run.smithi096:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-18T07:46:25.727 INFO:tasks.cephadm:Removing cluster... 2024-09-18T07:46:25.727 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 126f471a-758e-11ef-bceb-c7b262605968 --force 2024-09-18T07:46:26.088 DEBUG:teuthology.orchestra.run.smithi096:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 126f471a-758e-11ef-bceb-c7b262605968 --force 2024-09-18T07:46:26.501 INFO:tasks.cephadm:Removing cephadm ... 2024-09-18T07:46:26.502 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-18T07:46:26.517 DEBUG:teuthology.orchestra.run.smithi096:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-18T07:46:26.533 INFO:tasks.cephadm:Teardown complete 2024-09-18T07:46:26.533 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-18T07:46:26.549 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-18T07:46:26.550 DEBUG:teuthology.orchestra.run.smithi083:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T07:46:26.561 DEBUG:teuthology.orchestra.run.smithi096:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T07:46:26.572 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-09-18T07:46:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-18T07:46:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-09-18T07:46:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:^* hv01.front.sepia.ceph.com 3 6 377 51 -1698us[-1684us] +/- 56ms 2024-09-18T07:46:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 47 +538us[ +538us] +/- 56ms 2024-09-18T07:46:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 48 +1808us[+1808us] +/- 79ms 2024-09-18T07:46:26.577 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:46:26.586 INFO:teuthology.orchestra.run.smithi096.stderr:bash: line 1: ntpq: command not found 2024-09-18T07:46:26.592 INFO:teuthology.orchestra.run.smithi096.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-18T07:46:26.592 INFO:teuthology.orchestra.run.smithi096.stdout:=============================================================================== 2024-09-18T07:46:26.592 INFO:teuthology.orchestra.run.smithi096.stdout:^* hv01.front.sepia.ceph.com 3 6 377 44 -1718us[-1717us] +/- 56ms 2024-09-18T07:46:26.592 INFO:teuthology.orchestra.run.smithi096.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 44 +541us[ +542us] +/- 56ms 2024-09-18T07:46:26.593 INFO:teuthology.orchestra.run.smithi096.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 45 +1837us[+1838us] +/- 79ms 2024-09-18T07:46:26.593 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T07:46:26.593 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-18T07:46:26.605 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-18T07:46:26.606 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-18T07:46:26.669 DEBUG:teuthology.orchestra.run.smithi083:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-18T07:46:26.746 DEBUG:teuthology.orchestra.run.smithi096:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-18T07:46:26.817 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-18T07:46:26.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T07:46:26.846 DEBUG:teuthology.orchestra.run.smithi096:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-18T07:46:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T07:46:26.873 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-18T07:46:26.883 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-18T07:46:26.913 INFO:teuthology.task.internal:Duration was 1958.836367 seconds 2024-09-18T07:46:26.914 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-18T07:46:26.927 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-18T07:46:26.927 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-18T07:46:26.930 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-18T07:46:26.966 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-18T07:46:26.968 INFO:teuthology.orchestra.run.smithi096.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-18T07:46:27.410 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-18T07:46:27.411 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com 2024-09-18T07:46:27.411 DEBUG:teuthology.orchestra.run.smithi083:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-18T07:46:27.472 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi096.front.sepia.ceph.com 2024-09-18T07:46:27.472 DEBUG:teuthology.orchestra.run.smithi096:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-18T07:46:27.511 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-18T07:46:27.511 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-18T07:46:27.515 DEBUG:teuthology.orchestra.run.smithi096:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-18T07:46:27.580 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-18T07:46:27.580 DEBUG:teuthology.orchestra.run.smithi083:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-18T07:46:27.583 DEBUG:teuthology.orchestra.run.smithi096:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-18T07:46:41.557 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-18T07:46:41.573 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-18T07:46:41.574 DEBUG:teuthology.orchestra.run.smithi083:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-18T07:46:41.599 DEBUG:teuthology.orchestra.run.smithi096:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-18T07:46:41.630 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-18T07:46:41.641 DEBUG:teuthology.orchestra.run.smithi083:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-18T07:46:41.644 DEBUG:teuthology.orchestra.run.smithi096:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-18T07:46:41.665 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = core 2024-09-18T07:46:41.696 INFO:teuthology.orchestra.run.smithi096.stdout:kernel.core_pattern = core 2024-09-18T07:46:42.027 DEBUG:teuthology.orchestra.run.smithi083:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-18T07:46:42.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T07:46:42.042 DEBUG:teuthology.orchestra.run.smithi096:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-18T07:46:42.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T07:46:42.087 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-18T07:46:42.102 INFO:teuthology.task.internal:Transferring archived files... 2024-09-18T07:46:42.103 DEBUG:teuthology.misc:Transferring archived files from smithi083:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi/7910284/remote/smithi083 2024-09-18T07:46:42.103 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-18T07:46:42.163 DEBUG:teuthology.misc:Transferring archived files from smithi096:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-18_06:59:44-orch:cephadm-wip-guits-main-2024-09-17-1213-distro-default-smithi/7910284/remote/smithi096 2024-09-18T07:46:42.163 DEBUG:teuthology.orchestra.run.smithi096:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-18T07:46:42.217 INFO:teuthology.task.internal:Removing archive directory... 2024-09-18T07:46:42.217 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-18T07:46:42.220 DEBUG:teuthology.orchestra.run.smithi096:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-18T07:46:42.268 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-18T07:46:42.285 INFO:teuthology.task.internal:Not uploading archives. 2024-09-18T07:46:42.285 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-18T07:46:42.312 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-18T07:46:42.312 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-18T07:46:42.315 DEBUG:teuthology.orchestra.run.smithi096:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-18T07:46:42.327 INFO:teuthology.orchestra.run.smithi083.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 18 07:46 /home/ubuntu/cephtest 2024-09-18T07:46:42.332 INFO:teuthology.orchestra.run.smithi096.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 18 07:46 /home/ubuntu/cephtest 2024-09-18T07:46:42.333 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-18T07:46:42.345 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-18T07:46:42.446 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1958.836366891861 owner: scheduled_gabrioux@teuthology success: true 2024-09-18T07:46:42.446 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-18T07:46:42.535 INFO:teuthology.run:pass